| OLD | NEW |
| 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 import 'dart:convert'; | 5 import 'dart:convert'; |
| 6 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow | 6 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow |
| 7 // CommandOutput.exitCode in subclasses of CommandOutput. | 7 // CommandOutput.exitCode in subclasses of CommandOutput. |
| 8 import 'dart:io' as io; | 8 import 'dart:io' as io; |
| 9 | 9 |
| 10 import 'browser_controller.dart'; | 10 import 'browser_controller.dart'; |
| 11 import 'command.dart'; | 11 import 'command.dart'; |
| 12 import 'configuration.dart'; | 12 import 'configuration.dart'; |
| 13 import 'expectation.dart'; | 13 import 'expectation.dart'; |
| 14 import 'test_runner.dart'; | 14 import 'test_runner.dart'; |
| 15 import 'utils.dart'; | 15 import 'utils.dart'; |
| 16 | 16 |
| 17 /** | 17 /// CommandOutput records the output of a completed command: the process's exit |
| 18 * CommandOutput records the output of a completed command: the process's exit | 18 /// code, the standard output and standard error, whether the process timed out, |
| 19 * code, the standard output and standard error, whether the process timed out, | 19 /// and the time the process took to run. It does not contain a pointer to the |
| 20 * and the time the process took to run. It does not contain a pointer to the | 20 /// [TestCase] this is the output of, so some functions require the test case |
| 21 * [TestCase] this is the output of, so some functions require the test case | 21 /// to be passed as an argument. |
| 22 * to be passed as an argument. | 22 class CommandOutput extends UniqueObject { |
| 23 */ | 23 final Command command; |
| 24 abstract class CommandOutput { | |
| 25 Command get command; | |
| 26 | 24 |
| 27 Expectation result(TestCase testCase); | 25 final bool hasTimedOut; |
| 28 | 26 |
| 29 bool get hasCrashed; | 27 final Duration time; |
| 30 | 28 |
| 31 bool get hasTimedOut; | 29 final int exitCode; |
| 32 | 30 |
| 33 bool didFail(TestCase testCase); | 31 final int pid; |
| 34 | 32 |
| 35 bool hasFailed(TestCase testCase); | 33 final List<int> stdout; |
| 34 final List<int> stderr; |
| 36 | 35 |
| 37 bool get canRunDependendCommands; | 36 final bool compilationSkipped; |
| 38 | 37 |
| 39 bool get successful; // otherwise we might to retry running | 38 final List<String> diagnostics = []; |
| 40 | 39 |
| 41 Duration get time; | 40 CommandOutput(this.command, this.exitCode, this.hasTimedOut, this.stdout, |
| 42 | 41 this.stderr, this.time, this.compilationSkipped, this.pid); |
| 43 int get exitCode; | |
| 44 | |
| 45 int get pid; | |
| 46 | |
| 47 List<int> get stdout; | |
| 48 | |
| 49 List<int> get stderr; | |
| 50 | |
| 51 List<String> get diagnostics; | |
| 52 | |
| 53 bool get compilationSkipped; | |
| 54 } | |
| 55 | |
| 56 class CommandOutputImpl extends UniqueObject implements CommandOutput { | |
| 57 Command command; | |
| 58 int exitCode; | |
| 59 | |
| 60 bool timedOut; | |
| 61 List<int> stdout; | |
| 62 List<int> stderr; | |
| 63 Duration time; | |
| 64 List<String> diagnostics; | |
| 65 bool compilationSkipped; | |
| 66 int pid; | |
| 67 | |
| 68 /** | |
| 69 * A flag to indicate we have already printed a warning about ignoring the VM | |
| 70 * crash, to limit the amount of output produced per test. | |
| 71 */ | |
| 72 bool alreadyPrintedWarning = false; | |
| 73 | |
| 74 CommandOutputImpl( | |
| 75 Command this.command, | |
| 76 int this.exitCode, | |
| 77 bool this.timedOut, | |
| 78 List<int> this.stdout, | |
| 79 List<int> this.stderr, | |
| 80 Duration this.time, | |
| 81 bool this.compilationSkipped, | |
| 82 int this.pid) { | |
| 83 diagnostics = []; | |
| 84 } | |
| 85 | 42 |
| 86 Expectation result(TestCase testCase) { | 43 Expectation result(TestCase testCase) { |
| 87 if (hasCrashed) return Expectation.crash; | 44 if (hasCrashed) return Expectation.crash; |
| 88 if (hasTimedOut) return Expectation.timeout; | 45 if (hasTimedOut) return Expectation.timeout; |
| 89 if (hasFailed(testCase)) return Expectation.fail; | 46 if (hasFailed(testCase)) return Expectation.fail; |
| 90 if (hasNonUtf8) return Expectation.nonUtf8Error; | 47 if (hasNonUtf8) return Expectation.nonUtf8Error; |
| 48 |
| 91 return Expectation.pass; | 49 return Expectation.pass; |
| 92 } | 50 } |
| 93 | 51 |
| 94 bool get hasCrashed { | 52 bool get hasCrashed { |
| 95 // dart2js exits with code 253 in case of unhandled exceptions. | 53 // dart2js exits with code 253 in case of unhandled exceptions. |
| 96 // The dart binary exits with code 253 in case of an API error such | 54 // The dart binary exits with code 253 in case of an API error such |
| 97 // as an invalid snapshot file. | 55 // as an invalid snapshot file. |
| 98 // In either case an exit code of 253 is considered a crash. | 56 // In either case an exit code of 253 is considered a crash. |
| 99 if (exitCode == 253) return true; | 57 if (exitCode == 253) return true; |
| 100 if (io.Platform.operatingSystem == 'windows') { | 58 if (io.Platform.operatingSystem == 'windows') { |
| 101 // The VM uses std::abort to terminate on asserts. | 59 // The VM uses std::abort to terminate on asserts. |
| 102 // std::abort terminates with exit code 3 on Windows. | 60 // std::abort terminates with exit code 3 on Windows. |
| 103 if (exitCode == 3 || exitCode == CRASHING_BROWSER_EXITCODE) { | 61 if (exitCode == 3 || exitCode == browserCrashExitCode) { |
| 104 return !timedOut; | 62 return !hasTimedOut; |
| 105 } | 63 } |
| 106 // If a program receives an uncaught system exception, the program | 64 // If a program receives an uncaught system exception, the program |
| 107 // terminates with the exception code as exit code. | 65 // terminates with the exception code as exit code. |
| 108 // The 0x3FFFFF00 mask here tries to determine if an exception indicates | 66 // The 0x3FFFFF00 mask here tries to determine if an exception indicates |
| 109 // a crash of the program. | 67 // a crash of the program. |
| 110 // System exception codes can be found in 'winnt.h', for example | 68 // System exception codes can be found in 'winnt.h', for example |
| 111 // "#define STATUS_ACCESS_VIOLATION ((DWORD) 0xC0000005)" | 69 // "#define STATUS_ACCESS_VIOLATION ((DWORD) 0xC0000005)" |
| 112 return (!timedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0)); | 70 return (!hasTimedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0)); |
| 113 } | 71 } |
| 114 return !timedOut && ((exitCode < 0)); | 72 return !hasTimedOut && ((exitCode < 0)); |
| 115 } | 73 } |
| 116 | 74 |
| 117 bool get hasTimedOut => timedOut; | 75 bool _didFail(TestCase testCase) => exitCode != 0 && !hasCrashed; |
| 118 | |
| 119 bool didFail(TestCase testCase) { | |
| 120 return (exitCode != 0 && !hasCrashed); | |
| 121 } | |
| 122 | 76 |
| 123 bool get canRunDependendCommands { | 77 bool get canRunDependendCommands { |
| 124 // FIXME(kustermann): We may need to change this | 78 // FIXME(kustermann): We may need to change this |
| 125 return !hasTimedOut && exitCode == 0; | 79 return !hasTimedOut && exitCode == 0; |
| 126 } | 80 } |
| 127 | 81 |
| 128 bool get successful { | 82 bool get successful { |
| 129 // FIXME(kustermann): We may need to change this | 83 // FIXME(kustermann): We may need to change this |
| 130 return !hasTimedOut && exitCode == 0; | 84 return !hasTimedOut && exitCode == 0; |
| 131 } | 85 } |
| 132 | 86 |
| 87 // TODO(bob): Remove. |
| 133 // Reverse result of a negative test. | 88 // Reverse result of a negative test. |
| 134 bool hasFailed(TestCase testCase) { | 89 bool hasFailed(TestCase testCase) { |
| 135 return testCase.isNegative ? !didFail(testCase) : didFail(testCase); | 90 return testCase.isNegative ? !_didFail(testCase) : _didFail(testCase); |
| 136 } | 91 } |
| 137 | 92 |
| 138 bool get hasNonUtf8 => exitCode == NON_UTF_FAKE_EXITCODE; | 93 bool get hasNonUtf8 => exitCode == nonUtfFakeExitCode; |
| 139 | 94 |
| 140 Expectation _negateOutcomeIfNegativeTest( | 95 Expectation _negateOutcomeIfNegativeTest( |
| 141 Expectation outcome, bool isNegative) { | 96 Expectation outcome, bool isNegative) { |
| 142 if (!isNegative) return outcome; | 97 if (!isNegative) return outcome; |
| 143 if (outcome == Expectation.ignore) return outcome; | 98 if (outcome == Expectation.ignore) return outcome; |
| 144 if (outcome.canBeOutcomeOf(Expectation.fail)) { | 99 if (outcome.canBeOutcomeOf(Expectation.fail)) { |
| 145 return Expectation.pass; | 100 return Expectation.pass; |
| 146 } | 101 } |
| 147 return Expectation.fail; | 102 return Expectation.fail; |
| 148 } | 103 } |
| 149 } | 104 } |
| 150 | 105 |
| 151 class ContentShellCommandOutputImpl extends CommandOutputImpl { | 106 class ContentShellCommandOutput extends CommandOutput { |
| 152 // Although tests are reported as passing, content shell sometimes exits with | 107 // Although tests are reported as passing, content shell sometimes exits with |
| 153 // a nonzero exitcode which makes our dartium builders extremely falky. | 108 // a nonzero exitcode which makes our dartium builders extremely falky. |
| 154 // See: http://dartbug.com/15139. | 109 // See: http://dartbug.com/15139. |
| 155 // TODO(rnystrom): Is this still needed? The underlying bug is closed. | 110 // TODO(rnystrom): Is this still needed? The underlying bug is closed. |
| 156 static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022; | 111 static const _whitelistedContentShellExitCode = -1073740022; |
| 157 static bool isWindows = io.Platform.operatingSystem == 'windows'; | 112 |
| 158 static bool _failedBecauseOfFlakyInfrastructure( | 113 static bool _failedBecauseOfFlakyInfrastructure( |
| 159 Command command, bool timedOut, List<int> stderrBytes) { | 114 Command command, bool timedOut, List<int> stderrBytes) { |
| 160 // If the browser test failed, it may have been because content shell | 115 // If the browser test failed, it may have been because content shell |
| 161 // and the virtual framebuffer X server didn't hook up, or it crashed with | 116 // and the virtual framebuffer X server didn't hook up, or it crashed with |
| 162 // a core dump. Sometimes content shell crashes after it has set the stdout | 117 // a core dump. Sometimes content shell crashes after it has set the stdout |
| 163 // to PASS, so we have to do this check first. | 118 // to PASS, so we have to do this check first. |
| 164 // Content shell also fails with a broken pipe message: Issue 26739 | 119 // Content shell also fails with a broken pipe message: Issue 26739 |
| 165 var zygoteCrash = | 120 var zygoteCrash = |
| 166 new RegExp(r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe"); | 121 new RegExp(r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe"); |
| 167 var stderr = decodeUtf8(stderrBytes); | 122 var stderr = decodeUtf8(stderrBytes); |
| 168 // TODO(7564): See http://dartbug.com/7564 | 123 // TODO(7564): See http://dartbug.com/7564 |
| 169 // This may not be happening anymore. Test by removing this suppression. | 124 // This may not be happening anymore. Test by removing this suppression. |
| 170 if (stderr.contains(MESSAGE_CANNOT_OPEN_DISPLAY) || | 125 if (stderr.contains(cannotOpenDisplayMessage) || |
| 171 stderr.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) { | 126 stderr.contains(failedToRunCommandMessage)) { |
| 172 DebugLogger.warning( | 127 DebugLogger.warning( |
| 173 "Warning: Failure because of missing XDisplay. Test ignored"); | 128 "Warning: Failure because of missing XDisplay. Test ignored"); |
| 174 return true; | 129 return true; |
| 175 } | 130 } |
| 176 // TODO(26739): See http://dartbug.com/26739 | 131 // TODO(26739): See http://dartbug.com/26739 |
| 177 if (zygoteCrash.hasMatch(stderr)) { | 132 if (zygoteCrash.hasMatch(stderr)) { |
| 178 DebugLogger.warning("Warning: Failure because of content_shell " | 133 DebugLogger.warning("Warning: Failure because of content_shell " |
| 179 "zygote crash. Test ignored"); | 134 "zygote crash. Test ignored"); |
| 180 return true; | 135 return true; |
| 181 } | 136 } |
| 182 return false; | 137 return false; |
| 183 } | 138 } |
| 184 | 139 |
| 185 bool _infraFailure; | 140 final bool _infraFailure; |
| 186 | 141 |
| 187 ContentShellCommandOutputImpl( | 142 ContentShellCommandOutput( |
| 188 Command command, | 143 Command command, |
| 189 int exitCode, | 144 int exitCode, |
| 190 bool timedOut, | 145 bool timedOut, |
| 191 List<int> stdout, | 146 List<int> stdout, |
| 192 List<int> stderr, | 147 List<int> stderr, |
| 193 Duration time, | 148 Duration time, |
| 194 bool compilationSkipped) | 149 bool compilationSkipped) |
| 195 : _infraFailure = | 150 : _infraFailure = |
| 196 _failedBecauseOfFlakyInfrastructure(command, timedOut, stderr), | 151 _failedBecauseOfFlakyInfrastructure(command, timedOut, stderr), |
| 197 super(command, exitCode, timedOut, stdout, stderr, time, | 152 super(command, exitCode, timedOut, stdout, stderr, time, |
| (...skipping 21 matching lines...) Expand all Loading... |
| 219 return Expectation.fail; | 174 return Expectation.fail; |
| 220 } | 175 } |
| 221 return outcome; | 176 return outcome; |
| 222 } | 177 } |
| 223 | 178 |
| 224 bool get successful => canRunDependendCommands; | 179 bool get successful => canRunDependendCommands; |
| 225 | 180 |
| 226 bool get canRunDependendCommands { | 181 bool get canRunDependendCommands { |
| 227 // We cannot rely on the exit code of content_shell as a method to | 182 // We cannot rely on the exit code of content_shell as a method to |
| 228 // determine if we were successful or not. | 183 // determine if we were successful or not. |
| 229 return super.canRunDependendCommands && !didFail(null); | 184 return super.canRunDependendCommands && !_didFail(null); |
| 230 } | 185 } |
| 231 | 186 |
| 232 bool get hasCrashed { | 187 bool get hasCrashed => super.hasCrashed || _rendererCrashed; |
| 233 return super.hasCrashed || _rendererCrashed; | |
| 234 } | |
| 235 | 188 |
| 236 Expectation _getOutcome() { | 189 Expectation _getOutcome() { |
| 237 if (_browserTestFailure) { | 190 if (_browserTestFailure) { |
| 238 return Expectation.runtimeError; | 191 return Expectation.runtimeError; |
| 239 } | 192 } |
| 240 return Expectation.pass; | 193 return Expectation.pass; |
| 241 } | 194 } |
| 242 | 195 |
| 243 bool get _rendererCrashed => | 196 bool get _rendererCrashed => |
| 244 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); | 197 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); |
| 245 | 198 |
| 246 bool get _browserTestFailure { | 199 bool get _browserTestFailure { |
| 247 // Browser tests fail unless stdout contains | 200 // Browser tests fail unless stdout contains |
| 248 // 'Content-Type: text/plain' followed by 'PASS'. | 201 // 'Content-Type: text/plain' followed by 'PASS'. |
| 249 bool hasContentType = false; | 202 var hasContentType = false; |
| 250 var stdoutLines = decodeUtf8(super.stdout).split("\n"); | 203 var stdoutLines = decodeUtf8(super.stdout).split("\n"); |
| 251 var containsFail = false; | 204 var containsFail = false; |
| 252 var containsPass = false; | 205 var containsPass = false; |
| 253 for (String line in stdoutLines) { | 206 for (String line in stdoutLines) { |
| 254 switch (line) { | 207 switch (line) { |
| 255 case 'Content-Type: text/plain': | 208 case 'Content-Type: text/plain': |
| 256 hasContentType = true; | 209 hasContentType = true; |
| 257 break; | 210 break; |
| 258 case 'FAIL': | 211 case 'FAIL': |
| 259 if (hasContentType) { | 212 if (hasContentType) { |
| (...skipping 20 matching lines...) Expand all Loading... |
| 280 return true; | 233 return true; |
| 281 } | 234 } |
| 282 assert(containsPass); | 235 assert(containsPass); |
| 283 if (exitCode != 0) { | 236 if (exitCode != 0) { |
| 284 var message = "All tests passed, but exitCode != 0. " | 237 var message = "All tests passed, but exitCode != 0. " |
| 285 "Actual exitcode: $exitCode. " | 238 "Actual exitcode: $exitCode. " |
| 286 "($command)"; | 239 "($command)"; |
| 287 DebugLogger.warning(message); | 240 DebugLogger.warning(message); |
| 288 diagnostics.add(message); | 241 diagnostics.add(message); |
| 289 } | 242 } |
| 243 |
| 244 var isWindows = io.Platform.operatingSystem == 'windows'; |
| 290 return (!hasCrashed && | 245 return (!hasCrashed && |
| 291 exitCode != 0 && | 246 exitCode != 0 && |
| 292 (!isWindows || exitCode != WHITELISTED_CONTENTSHELL_EXITCODE)); | 247 (!isWindows || exitCode != _whitelistedContentShellExitCode)); |
| 293 } | 248 } |
| 249 |
| 294 DebugLogger.warning("Couldn't find 'Content-Type: text/plain' in output. " | 250 DebugLogger.warning("Couldn't find 'Content-Type: text/plain' in output. " |
| 295 "($command)."); | 251 "($command)."); |
| 296 return true; | 252 return true; |
| 297 } | 253 } |
| 298 } | 254 } |
| 299 | 255 |
| 300 class HTMLBrowserCommandOutputImpl extends ContentShellCommandOutputImpl { | 256 class HtmlBrowserCommandOutput extends ContentShellCommandOutput { |
| 301 HTMLBrowserCommandOutputImpl( | 257 HtmlBrowserCommandOutput( |
| 302 Command command, | 258 Command command, |
| 303 int exitCode, | 259 int exitCode, |
| 304 bool timedOut, | 260 bool timedOut, |
| 305 List<int> stdout, | 261 List<int> stdout, |
| 306 List<int> stderr, | 262 List<int> stderr, |
| 307 Duration time, | 263 Duration time, |
| 308 bool compilationSkipped) | 264 bool compilationSkipped) |
| 309 : super(command, exitCode, timedOut, stdout, stderr, time, | 265 : super(command, exitCode, timedOut, stdout, stderr, time, |
| 310 compilationSkipped); | 266 compilationSkipped); |
| 311 | 267 |
| 312 bool didFail(TestCase testCase) { | 268 bool _didFail(TestCase testCase) { |
| 313 return _getOutcome() != Expectation.pass; | 269 return _getOutcome() != Expectation.pass; |
| 314 } | 270 } |
| 315 | 271 |
| 316 bool get _browserTestFailure { | 272 bool get _browserTestFailure { |
| 317 // We should not need to convert back and forward. | 273 // We should not need to convert back and forward. |
| 318 var output = decodeUtf8(super.stdout); | 274 var output = decodeUtf8(super.stdout); |
| 319 if (output.contains("FAIL")) return true; | 275 if (output.contains("FAIL")) return true; |
| 320 return !output.contains("PASS"); | 276 return !output.contains("PASS"); |
| 321 } | 277 } |
| 322 } | 278 } |
| 323 | 279 |
| 324 class BrowserTestJsonResult { | 280 class BrowserTestJsonResult { |
| 325 static const ALLOWED_TYPES = const [ | 281 static const _allowedTypes = const [ |
| 326 'sync_exception', | 282 'sync_exception', |
| 327 'window_onerror', | 283 'window_onerror', |
| 328 'script_onerror', | 284 'script_onerror', |
| 329 'window_compilationerror', | 285 'window_compilationerror', |
| 330 'print', | 286 'print', |
| 331 'message_received', | 287 'message_received', |
| 332 'dom', | 288 'dom', |
| 333 'debug' | 289 'debug' |
| 334 ]; | 290 ]; |
| 335 | 291 |
| (...skipping 10 matching lines...) Expand all Loading... |
| 346 "$content"; | 302 "$content"; |
| 347 } | 303 } |
| 348 } | 304 } |
| 349 | 305 |
| 350 try { | 306 try { |
| 351 var events = JSON.decode(content); | 307 var events = JSON.decode(content); |
| 352 if (events != null) { | 308 if (events != null) { |
| 353 validate("Message must be a List", events is List); | 309 validate("Message must be a List", events is List); |
| 354 | 310 |
| 355 var messagesByType = <String, List<String>>{}; | 311 var messagesByType = <String, List<String>>{}; |
| 356 ALLOWED_TYPES.forEach((type) => messagesByType[type] = <String>[]); | 312 _allowedTypes.forEach((type) => messagesByType[type] = <String>[]); |
| 357 | 313 |
| 358 for (var entry in events) { | 314 for (var entry in events) { |
| 359 validate("An entry must be a Map", entry is Map); | 315 validate("An entry must be a Map", entry is Map); |
| 360 | 316 |
| 361 var type = entry['type']; | 317 var type = entry['type']; |
| 362 var value = entry['value'] as String; | 318 var value = entry['value'] as String; |
| 363 var timestamp = entry['timestamp']; | 319 var timestamp = entry['timestamp']; |
| 364 | 320 |
| 365 validate("'type' of an entry must be a String", type is String); | 321 validate("'type' of an entry must be a String", type is String); |
| 366 validate("'type' has to be in $ALLOWED_TYPES.", | 322 validate("'type' has to be in $_allowedTypes.", |
| 367 ALLOWED_TYPES.contains(type)); | 323 _allowedTypes.contains(type)); |
| 368 validate( | 324 validate( |
| 369 "'timestamp' of an entry must be a number", timestamp is num); | 325 "'timestamp' of an entry must be a number", timestamp is num); |
| 370 | 326 |
| 371 messagesByType[type].add(value); | 327 messagesByType[type].add(value); |
| 372 } | 328 } |
| 373 validate("The message must have exactly one 'dom' entry.", | 329 validate("The message must have exactly one 'dom' entry.", |
| 374 messagesByType['dom'].length == 1); | 330 messagesByType['dom'].length == 1); |
| 375 | 331 |
| 376 var dom = messagesByType['dom'][0]; | 332 var dom = messagesByType['dom'][0]; |
| 377 if (dom.endsWith('\n')) { | 333 if (dom.endsWith('\n')) { |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 436 var mainDone = | 392 var mainDone = |
| 437 searchForMsg(['print', 'message_received'], 'dart-main-done'); | 393 searchForMsg(['print', 'message_received'], 'dart-main-done'); |
| 438 | 394 |
| 439 if (mainStarted && mainDone) { | 395 if (mainStarted && mainDone) { |
| 440 return Expectation.pass; | 396 return Expectation.pass; |
| 441 } | 397 } |
| 442 return Expectation.fail; | 398 return Expectation.fail; |
| 443 } | 399 } |
| 444 } | 400 } |
| 445 | 401 |
| 446 class BrowserCommandOutputImpl extends CommandOutputImpl | 402 class BrowserCommandOutput extends CommandOutput |
| 447 with UnittestSuiteMessagesMixin { | 403 with UnittestSuiteMessagesMixin { |
| 448 BrowserTestOutput _result; | 404 final BrowserTestOutput _result; |
| 449 Expectation _rawOutcome; | 405 final Expectation _rawOutcome; |
| 450 | 406 |
| 451 factory BrowserCommandOutputImpl(Command command, BrowserTestOutput result) { | 407 factory BrowserCommandOutput(Command command, BrowserTestOutput result) { |
| 452 String indent(String string, int numSpaces) { | 408 String indent(String string, int numSpaces) { |
| 453 var spaces = new List.filled(numSpaces, ' ').join(''); | 409 var spaces = new List.filled(numSpaces, ' ').join(''); |
| 454 return string | 410 return string |
| 455 .replaceAll('\r\n', '\n') | 411 .replaceAll('\r\n', '\n') |
| 456 .split('\n') | 412 .split('\n') |
| 457 .map((line) => "$spaces$line") | 413 .map((line) => "$spaces$line") |
| 458 .join('\n'); | 414 .join('\n'); |
| 459 } | 415 } |
| 460 | 416 |
| 461 String stdout = ""; | 417 String stdout = ""; |
| (...skipping 30 matching lines...) Expand all Loading... |
| 492 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; | 448 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; |
| 493 } | 449 } |
| 494 | 450 |
| 495 stderr = '$stderr\n\n' | 451 stderr = '$stderr\n\n' |
| 496 'BrowserOutput while running the test (* EXPERIMENTAL *):\n' | 452 'BrowserOutput while running the test (* EXPERIMENTAL *):\n' |
| 497 'BrowserOutput.stdout:\n' | 453 'BrowserOutput.stdout:\n' |
| 498 '${indent(result.browserOutput.stdout.toString(), 2)}\n' | 454 '${indent(result.browserOutput.stdout.toString(), 2)}\n' |
| 499 'BrowserOutput.stderr:\n' | 455 'BrowserOutput.stderr:\n' |
| 500 '${indent(result.browserOutput.stderr.toString(), 2)}\n' | 456 '${indent(result.browserOutput.stderr.toString(), 2)}\n' |
| 501 '\n'; | 457 '\n'; |
| 502 return new BrowserCommandOutputImpl._internal( | 458 return new BrowserCommandOutput._internal( |
| 503 command, result, outcome, encodeUtf8(stdout), encodeUtf8(stderr)); | 459 command, result, outcome, encodeUtf8(stdout), encodeUtf8(stderr)); |
| 504 } | 460 } |
| 505 | 461 |
| 506 BrowserCommandOutputImpl._internal(Command command, BrowserTestOutput result, | 462 BrowserCommandOutput._internal(Command command, BrowserTestOutput result, |
| 507 this._rawOutcome, List<int> stdout, List<int> stderr) | 463 this._rawOutcome, List<int> stdout, List<int> stderr) |
| 508 : super(command, 0, result.didTimeout, stdout, stderr, result.duration, | 464 : _result = result, |
| 509 false, 0) { | 465 super(command, 0, result.didTimeout, stdout, stderr, result.duration, |
| 510 _result = result; | 466 false, 0); |
| 511 } | |
| 512 | 467 |
| 513 Expectation result(TestCase testCase) { | 468 Expectation result(TestCase testCase) { |
| 514 // Handle timeouts first | 469 // Handle timeouts first. |
| 515 if (_result.didTimeout) { | 470 if (_result.didTimeout) { |
| 516 if (testCase.configuration.runtime == Runtime.ie11) { | 471 if (testCase.configuration.runtime == Runtime.ie11) { |
| 517 // TODO(28955): See http://dartbug.com/28955 | 472 // TODO(28955): See http://dartbug.com/28955 |
| 518 DebugLogger.warning("Timeout of ie11 on test ${testCase.displayName}"); | 473 DebugLogger.warning("Timeout of ie11 on test ${testCase.displayName}"); |
| 519 return Expectation.ignore; | 474 return Expectation.ignore; |
| 520 } | 475 } |
| 521 return Expectation.timeout; | 476 return Expectation.timeout; |
| 522 } | 477 } |
| 523 | 478 |
| 524 if (hasNonUtf8) return Expectation.nonUtf8Error; | 479 if (hasNonUtf8) return Expectation.nonUtf8Error; |
| 525 | 480 |
| 526 // Multitests are handled specially | 481 // Multitests are handled specially. |
| 527 if (testCase.hasRuntimeError) { | 482 if (testCase.hasRuntimeError) { |
| 528 if (_rawOutcome == Expectation.runtimeError) return Expectation.pass; | 483 if (_rawOutcome == Expectation.runtimeError) return Expectation.pass; |
| 529 return Expectation.missingRuntimeError; | 484 return Expectation.missingRuntimeError; |
| 530 } | 485 } |
| 531 | 486 |
| 532 return _negateOutcomeIfNegativeTest(_rawOutcome, testCase.isNegative); | 487 return _negateOutcomeIfNegativeTest(_rawOutcome, testCase.isNegative); |
| 533 } | 488 } |
| 534 } | 489 } |
| 535 | 490 |
| 536 class AnalysisCommandOutputImpl extends CommandOutputImpl { | 491 class AnalysisCommandOutput extends CommandOutput { |
| 537 // An error line has 8 fields that look like: | 492 // An error line has 8 fields that look like: |
| 538 // ERROR|COMPILER|MISSING_SOURCE|file:/tmp/t.dart|15|1|24|Missing source. | 493 // ERROR|COMPILER|MISSING_SOURCE|file:/tmp/t.dart|15|1|24|Missing source. |
| 539 final int ERROR_LEVEL = 0; | 494 static const int _errorLevel = 0; |
| 540 final int ERROR_TYPE = 1; | 495 static const int _formattedError = 7; |
| 541 final int FILENAME = 3; | |
| 542 final int FORMATTED_ERROR = 7; | |
| 543 | 496 |
| 544 AnalysisCommandOutputImpl( | 497 AnalysisCommandOutput( |
| 545 Command command, | 498 Command command, |
| 546 int exitCode, | 499 int exitCode, |
| 547 bool timedOut, | 500 bool timedOut, |
| 548 List<int> stdout, | 501 List<int> stdout, |
| 549 List<int> stderr, | 502 List<int> stderr, |
| 550 Duration time, | 503 Duration time, |
| 551 bool compilationSkipped) | 504 bool compilationSkipped) |
| 552 : super(command, exitCode, timedOut, stdout, stderr, time, | 505 : super(command, exitCode, timedOut, stdout, stderr, time, |
| 553 compilationSkipped, 0); | 506 compilationSkipped, 0); |
| 554 | 507 |
| 555 Expectation result(TestCase testCase) { | 508 Expectation result(TestCase testCase) { |
| 556 // TODO(kustermann): If we run the analyzer not in batch mode, make sure | 509 // TODO(kustermann): If we run the analyzer not in batch mode, make sure |
| 557 // that command.exitCodes matches 2 (errors), 1 (warnings), 0 (no warnings, | 510 // that command.exitCodes matches 2 (errors), 1 (warnings), 0 (no warnings, |
| 558 // no errors) | 511 // no errors) |
| 559 | 512 |
| 560 // Handle crashes and timeouts first | 513 // Handle crashes and timeouts first |
| 561 if (hasCrashed) return Expectation.crash; | 514 if (hasCrashed) return Expectation.crash; |
| 562 if (hasTimedOut) return Expectation.timeout; | 515 if (hasTimedOut) return Expectation.timeout; |
| 563 if (hasNonUtf8) return Expectation.nonUtf8Error; | 516 if (hasNonUtf8) return Expectation.nonUtf8Error; |
| 564 | 517 |
| 565 // Get the errors/warnings from the analyzer | 518 // Get the errors/warnings from the analyzer |
| 566 List<String> errors = []; | 519 var errors = <String>[]; |
| 567 List<String> warnings = []; | 520 var warnings = <String>[]; |
| 568 parseAnalyzerOutput(errors, warnings); | 521 parseAnalyzerOutput(errors, warnings); |
| 569 | 522 |
| 570 // Handle errors / missing errors | 523 // Handle errors / missing errors |
| 571 if (testCase.expectCompileError) { | 524 if (testCase.expectCompileError) { |
| 572 if (errors.length > 0) { | 525 if (errors.isNotEmpty) { |
| 573 return Expectation.pass; | 526 return Expectation.pass; |
| 574 } | 527 } |
| 575 return Expectation.missingCompileTimeError; | 528 return Expectation.missingCompileTimeError; |
| 576 } | 529 } |
| 577 if (errors.length > 0) { | 530 if (errors.isNotEmpty) { |
| 578 return Expectation.compileTimeError; | 531 return Expectation.compileTimeError; |
| 579 } | 532 } |
| 580 | 533 |
| 581 // Handle static warnings / missing static warnings | 534 // Handle static warnings / missing static warnings |
| 582 if (testCase.hasStaticWarning) { | 535 if (testCase.hasStaticWarning) { |
| 583 if (warnings.length > 0) { | 536 if (warnings.isNotEmpty) { |
| 584 return Expectation.pass; | 537 return Expectation.pass; |
| 585 } | 538 } |
| 586 return Expectation.missingStaticWarning; | 539 return Expectation.missingStaticWarning; |
| 587 } | 540 } |
| 588 if (warnings.length > 0) { | 541 if (warnings.isNotEmpty) { |
| 589 return Expectation.staticWarning; | 542 return Expectation.staticWarning; |
| 590 } | 543 } |
| 591 | 544 |
| 592 assert(errors.length == 0 && warnings.length == 0); | 545 assert(errors.isEmpty && warnings.isEmpty); |
| 593 assert(!testCase.hasCompileError && !testCase.hasStaticWarning); | 546 assert(!testCase.hasCompileError && !testCase.hasStaticWarning); |
| 594 return Expectation.pass; | 547 return Expectation.pass; |
| 595 } | 548 } |
| 596 | 549 |
| 597 void parseAnalyzerOutput(List<String> outErrors, List<String> outWarnings) { | 550 void parseAnalyzerOutput(List<String> outErrors, List<String> outWarnings) { |
| 598 // Parse a line delimited by the | character using \ as an escape character | 551 // Parse a line delimited by the | character using \ as an escape character |
| 599 // like: FOO|BAR|FOO\|BAR|FOO\\BAZ as 4 fields: FOO BAR FOO|BAR FOO\BAZ | 552 // like: FOO|BAR|FOO\|BAR|FOO\\BAZ as 4 fields: FOO BAR FOO|BAR FOO\BAZ |
| 600 List<String> splitMachineError(String line) { | 553 List<String> splitMachineError(String line) { |
| 601 StringBuffer field = new StringBuffer(); | 554 var field = new StringBuffer(); |
| 602 List<String> result = []; | 555 var result = <String>[]; |
| 603 bool escaped = false; | 556 var escaped = false; |
| 604 for (var i = 0; i < line.length; i++) { | 557 for (var i = 0; i < line.length; i++) { |
| 605 var c = line[i]; | 558 var c = line[i]; |
| 606 if (!escaped && c == '\\') { | 559 if (!escaped && c == '\\') { |
| 607 escaped = true; | 560 escaped = true; |
| 608 continue; | 561 continue; |
| 609 } | 562 } |
| 610 escaped = false; | 563 escaped = false; |
| 611 if (c == '|') { | 564 if (c == '|') { |
| 612 result.add(field.toString()); | 565 result.add(field.toString()); |
| 613 field = new StringBuffer(); | 566 field = new StringBuffer(); |
| 614 continue; | 567 continue; |
| 615 } | 568 } |
| 616 field.write(c); | 569 field.write(c); |
| 617 } | 570 } |
| 618 result.add(field.toString()); | 571 result.add(field.toString()); |
| 619 return result; | 572 return result; |
| 620 } | 573 } |
| 621 | 574 |
| 622 for (String line in decodeUtf8(super.stderr).split("\n")) { | 575 for (String line in decodeUtf8(super.stderr).split("\n")) { |
| 623 if (line.length == 0) continue; | 576 if (line.isEmpty) continue; |
| 577 |
| 624 List<String> fields = splitMachineError(line); | 578 List<String> fields = splitMachineError(line); |
| 625 // We only consider errors/warnings for files of interest. | 579 // We only consider errors/warnings for files of interest. |
| 626 if (fields.length > FORMATTED_ERROR) { | 580 if (fields.length > _formattedError) { |
| 627 if (fields[ERROR_LEVEL] == 'ERROR') { | 581 if (fields[_errorLevel] == 'ERROR') { |
| 628 outErrors.add(fields[FORMATTED_ERROR]); | 582 outErrors.add(fields[_formattedError]); |
| 629 } else if (fields[ERROR_LEVEL] == 'WARNING') { | 583 } else if (fields[_errorLevel] == 'WARNING') { |
| 630 outWarnings.add(fields[FORMATTED_ERROR]); | 584 outWarnings.add(fields[_formattedError]); |
| 631 } | 585 } |
| 632 // OK to Skip error output that doesn't match the machine format | 586 // OK to Skip error output that doesn't match the machine format. |
| 633 } | 587 } |
| 634 } | 588 } |
| 635 } | 589 } |
| 636 } | 590 } |
| 637 | 591 |
| 638 class VmCommandOutputImpl extends CommandOutputImpl | 592 class VMCommandOutput extends CommandOutput with UnittestSuiteMessagesMixin { |
| 639 with UnittestSuiteMessagesMixin { | 593 static const _dfeErrorExitCode = 252; |
| 640 static const DART_VM_EXITCODE_DFE_ERROR = 252; | 594 static const _compileErrorExitCode = 254; |
| 641 static const DART_VM_EXITCODE_COMPILE_TIME_ERROR = 254; | 595 static const _uncaughtExceptionExitCode = 255; |
| 642 static const DART_VM_EXITCODE_UNCAUGHT_EXCEPTION = 255; | |
| 643 | 596 |
| 644 VmCommandOutputImpl(Command command, int exitCode, bool timedOut, | 597 VMCommandOutput(Command command, int exitCode, bool timedOut, |
| 645 List<int> stdout, List<int> stderr, Duration time, int pid) | 598 List<int> stdout, List<int> stderr, Duration time, int pid) |
| 646 : super(command, exitCode, timedOut, stdout, stderr, time, false, pid); | 599 : super(command, exitCode, timedOut, stdout, stderr, time, false, pid); |
| 647 | 600 |
| 648 Expectation result(TestCase testCase) { | 601 Expectation result(TestCase testCase) { |
| 649 // Handle crashes and timeouts first | 602 // Handle crashes and timeouts first. |
| 650 if (exitCode == DART_VM_EXITCODE_DFE_ERROR) return Expectation.dartkCrash; | 603 if (exitCode == _dfeErrorExitCode) return Expectation.dartkCrash; |
| 651 if (hasCrashed) return Expectation.crash; | 604 if (hasCrashed) return Expectation.crash; |
| 652 if (hasTimedOut) return Expectation.timeout; | 605 if (hasTimedOut) return Expectation.timeout; |
| 653 if (hasNonUtf8) return Expectation.nonUtf8Error; | 606 if (hasNonUtf8) return Expectation.nonUtf8Error; |
| 654 | 607 |
| 655 // Multitests are handled specially | 608 // Multitests are handled specially. |
| 656 if (testCase.expectCompileError) { | 609 if (testCase.expectCompileError) { |
| 657 if (exitCode == DART_VM_EXITCODE_COMPILE_TIME_ERROR) { | 610 if (exitCode == _compileErrorExitCode) { |
| 658 return Expectation.pass; | 611 return Expectation.pass; |
| 659 } | 612 } |
| 660 return Expectation.missingCompileTimeError; | 613 return Expectation.missingCompileTimeError; |
| 661 } | 614 } |
| 615 |
| 662 if (testCase.hasRuntimeError) { | 616 if (testCase.hasRuntimeError) { |
| 663 // TODO(kustermann): Do we consider a "runtimeError" only an uncaught | 617 // TODO(kustermann): Do we consider a "runtimeError" only an uncaught |
| 664 // exception or does any nonzero exit code fullfil this requirement? | 618 // exception or does any nonzero exit code fullfil this requirement? |
| 665 if (exitCode != 0) { | 619 if (exitCode != 0) { |
| 666 return Expectation.pass; | 620 return Expectation.pass; |
| 667 } | 621 } |
| 668 return Expectation.missingRuntimeError; | 622 return Expectation.missingRuntimeError; |
| 669 } | 623 } |
| 670 | 624 |
| 671 // The actual outcome depends on the exitCode | 625 // The actual outcome depends on the exitCode. |
| 672 Expectation outcome; | 626 var outcome = Expectation.pass; |
| 673 if (exitCode == DART_VM_EXITCODE_COMPILE_TIME_ERROR) { | 627 if (exitCode == _compileErrorExitCode) { |
| 674 outcome = Expectation.compileTimeError; | 628 outcome = Expectation.compileTimeError; |
| 675 } else if (exitCode == DART_VM_EXITCODE_UNCAUGHT_EXCEPTION) { | 629 } else if (exitCode == _uncaughtExceptionExitCode) { |
| 676 outcome = Expectation.runtimeError; | 630 outcome = Expectation.runtimeError; |
| 677 } else if (exitCode != 0) { | 631 } else if (exitCode != 0) { |
| 678 // This is a general fail, in case we get an unknown nonzero exitcode. | 632 // This is a general fail, in case we get an unknown nonzero exitcode. |
| 679 outcome = Expectation.fail; | 633 outcome = Expectation.fail; |
| 680 } else { | |
| 681 outcome = Expectation.pass; | |
| 682 } | 634 } |
| 635 |
| 683 outcome = _negateOutcomeIfIncompleteAsyncTest(outcome, decodeUtf8(stdout)); | 636 outcome = _negateOutcomeIfIncompleteAsyncTest(outcome, decodeUtf8(stdout)); |
| 684 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); | 637 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); |
| 685 } | 638 } |
| 686 } | 639 } |
| 687 | 640 |
| 688 class CompilationCommandOutputImpl extends CommandOutputImpl { | 641 class CompilationCommandOutput extends CommandOutput { |
| 689 static const DART2JS_EXITCODE_CRASH = 253; | 642 static const _crashExitCode = 253; |
| 690 | 643 |
| 691 CompilationCommandOutputImpl( | 644 CompilationCommandOutput( |
| 692 Command command, | 645 Command command, |
| 693 int exitCode, | 646 int exitCode, |
| 694 bool timedOut, | 647 bool timedOut, |
| 695 List<int> stdout, | 648 List<int> stdout, |
| 696 List<int> stderr, | 649 List<int> stderr, |
| 697 Duration time, | 650 Duration time, |
| 698 bool compilationSkipped) | 651 bool compilationSkipped) |
| 699 : super(command, exitCode, timedOut, stdout, stderr, time, | 652 : super(command, exitCode, timedOut, stdout, stderr, time, |
| 700 compilationSkipped, 0); | 653 compilationSkipped, 0); |
| 701 | 654 |
| 702 Expectation result(TestCase testCase) { | 655 Expectation result(TestCase testCase) { |
| 703 // Handle general crash/timeout detection. | 656 // Handle general crash/timeout detection. |
| 704 if (hasCrashed) return Expectation.crash; | 657 if (hasCrashed) return Expectation.crash; |
| 705 if (hasTimedOut) { | 658 if (hasTimedOut) { |
| 706 bool isWindows = io.Platform.operatingSystem == 'windows'; | 659 var isWindows = io.Platform.operatingSystem == 'windows'; |
| 707 bool isBrowserTestCase = | 660 var isBrowserTestCase = |
| 708 testCase.commands.any((command) => command is BrowserTestCommand); | 661 testCase.commands.any((command) => command is BrowserTestCommand); |
| 709 // TODO(26060) Dart2js batch mode hangs on Windows under heavy load. | 662 // TODO(26060) Dart2js batch mode hangs on Windows under heavy load. |
| 710 return (isWindows && isBrowserTestCase) | 663 return (isWindows && isBrowserTestCase) |
| 711 ? Expectation.ignore | 664 ? Expectation.ignore |
| 712 : Expectation.timeout; | 665 : Expectation.timeout; |
| 713 } | 666 } |
| 714 if (hasNonUtf8) return Expectation.nonUtf8Error; | 667 if (hasNonUtf8) return Expectation.nonUtf8Error; |
| 715 | 668 |
| 716 // Handle dart2js specific crash detection | 669 // Handle dart2js specific crash detection |
| 717 if (exitCode == DART2JS_EXITCODE_CRASH || | 670 if (exitCode == _crashExitCode || |
| 718 exitCode == VmCommandOutputImpl.DART_VM_EXITCODE_COMPILE_TIME_ERROR || | 671 exitCode == VMCommandOutput._compileErrorExitCode || |
| 719 exitCode == VmCommandOutputImpl.DART_VM_EXITCODE_UNCAUGHT_EXCEPTION) { | 672 exitCode == VMCommandOutput._uncaughtExceptionExitCode) { |
| 720 return Expectation.crash; | 673 return Expectation.crash; |
| 721 } | 674 } |
| 722 | 675 |
| 723 // Multitests are handled specially | 676 // Multitests are handled specially. |
| 724 if (testCase.expectCompileError) { | 677 if (testCase.expectCompileError) { |
| 725 // Nonzero exit code of the compiler means compilation failed | 678 // Nonzero exit code of the compiler means compilation failed |
| 726 // TODO(kustermann): Do we have a special exit code in that case??? | 679 // TODO(kustermann): Do we have a special exit code in that case??? |
| 727 if (exitCode != 0) { | 680 if (exitCode != 0) { |
| 728 return Expectation.pass; | 681 return Expectation.pass; |
| 729 } | 682 } |
| 730 return Expectation.missingCompileTimeError; | 683 return Expectation.missingCompileTimeError; |
| 731 } | 684 } |
| 732 | 685 |
| 733 // TODO(kustermann): This is a hack, remove it | 686 // TODO(kustermann): This is a hack, remove it. |
| 734 if (testCase.hasRuntimeError && testCase.commands.length > 1) { | 687 if (testCase.hasRuntimeError && testCase.commands.length > 1) { |
| 735 // We expected to run the test, but we got an compile time error. | 688 // We expected to run the test, but we got an compile time error. |
| 736 // If the compilation succeeded, we wouldn't be in here! | 689 // If the compilation succeeded, we wouldn't be in here! |
| 737 assert(exitCode != 0); | 690 assert(exitCode != 0); |
| 738 return Expectation.compileTimeError; | 691 return Expectation.compileTimeError; |
| 739 } | 692 } |
| 740 | 693 |
| 741 Expectation outcome = | 694 var outcome = |
| 742 exitCode == 0 ? Expectation.pass : Expectation.compileTimeError; | 695 exitCode == 0 ? Expectation.pass : Expectation.compileTimeError; |
| 743 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); | 696 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); |
| 744 } | 697 } |
| 745 } | 698 } |
| 746 | 699 |
| 747 class KernelCompilationCommandOutputImpl extends CompilationCommandOutputImpl { | 700 class KernelCompilationCommandOutput extends CompilationCommandOutput { |
| 748 KernelCompilationCommandOutputImpl( | 701 KernelCompilationCommandOutput( |
| 749 Command command, | 702 Command command, |
| 750 int exitCode, | 703 int exitCode, |
| 751 bool timedOut, | 704 bool timedOut, |
| 752 List<int> stdout, | 705 List<int> stdout, |
| 753 List<int> stderr, | 706 List<int> stderr, |
| 754 Duration time, | 707 Duration time, |
| 755 bool compilationSkipped) | 708 bool compilationSkipped) |
| 756 : super(command, exitCode, timedOut, stdout, stderr, time, | 709 : super(command, exitCode, timedOut, stdout, stderr, time, |
| 757 compilationSkipped); | 710 compilationSkipped); |
| 758 | 711 |
| 759 bool get canRunDependendCommands { | 712 bool get canRunDependendCommands { |
| 760 // See [BatchRunnerProcess]: 0 means success, 1 means compile-time error. | 713 // See [BatchRunnerProcess]: 0 means success, 1 means compile-time error. |
| 761 // TODO(asgerf): When the frontend supports it, continue running even if | 714 // TODO(asgerf): When the frontend supports it, continue running even if |
| 762 // there were compile-time errors. See kernel_sdk issue #18. | 715 // there were compile-time errors. See kernel_sdk issue #18. |
| 763 return !hasCrashed && !timedOut && exitCode == 0; | 716 return !hasCrashed && !hasTimedOut && exitCode == 0; |
| 764 } | 717 } |
| 765 | 718 |
| 766 Expectation result(TestCase testCase) { | 719 Expectation result(TestCase testCase) { |
| 767 Expectation result = super.result(testCase); | 720 Expectation result = super.result(testCase); |
| 768 if (result.canBeOutcomeOf(Expectation.crash)) { | 721 if (result.canBeOutcomeOf(Expectation.crash)) { |
| 769 return Expectation.dartkCrash; | 722 return Expectation.dartkCrash; |
| 770 } else if (result.canBeOutcomeOf(Expectation.timeout)) { | 723 } else if (result.canBeOutcomeOf(Expectation.timeout)) { |
| 771 return Expectation.dartkTimeout; | 724 return Expectation.dartkTimeout; |
| 772 } else if (result.canBeOutcomeOf(Expectation.compileTimeError)) { | 725 } else if (result.canBeOutcomeOf(Expectation.compileTimeError)) { |
| 773 return Expectation.dartkCompileTimeError; | 726 return Expectation.dartkCompileTimeError; |
| 774 } | 727 } |
| 775 return result; | 728 return result; |
| 776 } | 729 } |
| 777 | 730 |
| 778 // If the compiler was able to produce a Kernel IR file we want to run the | 731 /// If the compiler was able to produce a Kernel IR file we want to run the |
| 779 // result on the Dart VM. We therefore mark the [KernelCompilationCommand] as | 732 /// result on the Dart VM. We therefore mark the [KernelCompilationCommand] |
| 780 // successful. | 733 /// as successful. |
| 781 // => This ensures we test that the DartVM produces correct CompileTime errors | 734 /// |
| 782 // as it is supposed to for our test suites. | 735 /// This ensures we test that the DartVM produces correct CompileTime errors |
| 736 /// as it is supposed to for our test suites. |
| 783 bool get successful => canRunDependendCommands; | 737 bool get successful => canRunDependendCommands; |
| 784 } | 738 } |
| 785 | 739 |
| 786 class JsCommandlineOutputImpl extends CommandOutputImpl | 740 class JSCommandLineOutput extends CommandOutput |
| 787 with UnittestSuiteMessagesMixin { | 741 with UnittestSuiteMessagesMixin { |
| 788 JsCommandlineOutputImpl(Command command, int exitCode, bool timedOut, | 742 JSCommandLineOutput(Command command, int exitCode, bool timedOut, |
| 789 List<int> stdout, List<int> stderr, Duration time) | 743 List<int> stdout, List<int> stderr, Duration time) |
| 790 : super(command, exitCode, timedOut, stdout, stderr, time, false, 0); | 744 : super(command, exitCode, timedOut, stdout, stderr, time, false, 0); |
| 791 | 745 |
| 792 Expectation result(TestCase testCase) { | 746 Expectation result(TestCase testCase) { |
| 793 // Handle crashes and timeouts first | 747 // Handle crashes and timeouts first. |
| 794 if (hasCrashed) return Expectation.crash; | 748 if (hasCrashed) return Expectation.crash; |
| 795 if (hasTimedOut) return Expectation.timeout; | 749 if (hasTimedOut) return Expectation.timeout; |
| 796 if (hasNonUtf8) return Expectation.nonUtf8Error; | 750 if (hasNonUtf8) return Expectation.nonUtf8Error; |
| 797 | 751 |
| 798 if (testCase.hasRuntimeError) { | 752 if (testCase.hasRuntimeError) { |
| 799 if (exitCode != 0) return Expectation.pass; | 753 if (exitCode != 0) return Expectation.pass; |
| 800 return Expectation.missingRuntimeError; | 754 return Expectation.missingRuntimeError; |
| 801 } | 755 } |
| 802 | 756 |
| 803 var outcome = exitCode == 0 ? Expectation.pass : Expectation.runtimeError; | 757 var outcome = exitCode == 0 ? Expectation.pass : Expectation.runtimeError; |
| 804 outcome = _negateOutcomeIfIncompleteAsyncTest(outcome, decodeUtf8(stdout)); | 758 outcome = _negateOutcomeIfIncompleteAsyncTest(outcome, decodeUtf8(stdout)); |
| 805 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); | 759 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); |
| 806 } | 760 } |
| 807 } | 761 } |
| 808 | 762 |
| 809 class PubCommandOutputImpl extends CommandOutputImpl { | 763 class PubCommandOutput extends CommandOutput { |
| 810 PubCommandOutputImpl(PubCommand command, int exitCode, bool timedOut, | 764 PubCommandOutput(PubCommand command, int exitCode, bool timedOut, |
| 811 List<int> stdout, List<int> stderr, Duration time) | 765 List<int> stdout, List<int> stderr, Duration time) |
| 812 : super(command, exitCode, timedOut, stdout, stderr, time, false, 0); | 766 : super(command, exitCode, timedOut, stdout, stderr, time, false, 0); |
| 813 | 767 |
| 814 Expectation result(TestCase testCase) { | 768 Expectation result(TestCase testCase) { |
| 815 // Handle crashes and timeouts first | 769 // Handle crashes and timeouts first. |
| 816 if (hasCrashed) return Expectation.crash; | 770 if (hasCrashed) return Expectation.crash; |
| 817 if (hasTimedOut) return Expectation.timeout; | 771 if (hasTimedOut) return Expectation.timeout; |
| 818 if (hasNonUtf8) return Expectation.nonUtf8Error; | 772 if (hasNonUtf8) return Expectation.nonUtf8Error; |
| 819 | 773 |
| 820 if (exitCode == 0) { | 774 if (exitCode == 0) { |
| 821 return Expectation.pass; | 775 return Expectation.pass; |
| 822 } else if ((command as PubCommand).command == 'get') { | 776 } else if ((command as PubCommand).command == 'get') { |
| 823 return Expectation.pubGetError; | 777 return Expectation.pubGetError; |
| 824 } else { | 778 } else { |
| 825 return Expectation.fail; | 779 return Expectation.fail; |
| 826 } | 780 } |
| 827 } | 781 } |
| 828 } | 782 } |
| 829 | 783 |
| 830 class ScriptCommandOutputImpl extends CommandOutputImpl { | 784 class ScriptCommandOutput extends CommandOutput { |
| 831 final Expectation _result; | 785 final Expectation _result; |
| 832 | 786 |
| 833 ScriptCommandOutputImpl(ScriptCommand command, this._result, | 787 ScriptCommandOutput(ScriptCommand command, this._result, |
| 834 String scriptExecutionInformation, Duration time) | 788 String scriptExecutionInformation, Duration time) |
| 835 : super(command, 0, false, [], [], time, false, 0) { | 789 : super(command, 0, false, [], [], time, false, 0) { |
| 836 var lines = scriptExecutionInformation.split("\n"); | 790 var lines = scriptExecutionInformation.split("\n"); |
| 837 diagnostics.addAll(lines); | 791 diagnostics.addAll(lines); |
| 838 } | 792 } |
| 839 | 793 |
| 840 Expectation result(TestCase testCase) => _result; | 794 Expectation result(TestCase testCase) => _result; |
| 841 | 795 |
| 842 bool get canRunDependendCommands => _result == Expectation.pass; | 796 bool get canRunDependendCommands => _result == Expectation.pass; |
| 843 | 797 |
| 844 bool get successful => _result == Expectation.pass; | 798 bool get successful => _result == Expectation.pass; |
| 845 } | 799 } |
| 846 | 800 |
| 847 CommandOutput createCommandOutput(Command command, int exitCode, bool timedOut, | 801 CommandOutput createCommandOutput(Command command, int exitCode, bool timedOut, |
| 848 List<int> stdout, List<int> stderr, Duration time, bool compilationSkipped, | 802 List<int> stdout, List<int> stderr, Duration time, bool compilationSkipped, |
| 849 [int pid = 0]) { | 803 [int pid = 0]) { |
| 850 if (command is ContentShellCommand) { | 804 if (command is ContentShellCommand) { |
| 851 return new ContentShellCommandOutputImpl( | 805 return new ContentShellCommandOutput( |
| 852 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 806 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
| 853 } else if (command is BrowserTestCommand) { | 807 } else if (command is BrowserTestCommand) { |
| 854 return new HTMLBrowserCommandOutputImpl( | 808 return new HtmlBrowserCommandOutput( |
| 855 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 809 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
| 856 } else if (command is AnalysisCommand) { | 810 } else if (command is AnalysisCommand) { |
| 857 return new AnalysisCommandOutputImpl( | 811 return new AnalysisCommandOutput( |
| 858 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 812 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
| 859 } else if (command is VmCommand) { | 813 } else if (command is VmCommand) { |
| 860 return new VmCommandOutputImpl( | 814 return new VMCommandOutput( |
| 861 command, exitCode, timedOut, stdout, stderr, time, pid); | 815 command, exitCode, timedOut, stdout, stderr, time, pid); |
| 862 } else if (command is KernelCompilationCommand) { | 816 } else if (command is KernelCompilationCommand) { |
| 863 return new KernelCompilationCommandOutputImpl( | 817 return new KernelCompilationCommandOutput( |
| 864 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 818 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
| 865 } else if (command is AdbPrecompilationCommand) { | 819 } else if (command is AdbPrecompilationCommand) { |
| 866 return new VmCommandOutputImpl( | 820 return new VMCommandOutput( |
| 867 command, exitCode, timedOut, stdout, stderr, time, pid); | 821 command, exitCode, timedOut, stdout, stderr, time, pid); |
| 868 } else if (command is CompilationCommand) { | 822 } else if (command is CompilationCommand) { |
| 869 if (command.displayName == 'precompiler' || | 823 if (command.displayName == 'precompiler' || |
| 870 command.displayName == 'app_jit') { | 824 command.displayName == 'app_jit') { |
| 871 return new VmCommandOutputImpl( | 825 return new VMCommandOutput( |
| 872 command, exitCode, timedOut, stdout, stderr, time, pid); | 826 command, exitCode, timedOut, stdout, stderr, time, pid); |
| 873 } | 827 } |
| 874 return new CompilationCommandOutputImpl( | 828 return new CompilationCommandOutput( |
| 875 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 829 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
| 876 } else if (command is JSCommandlineCommand) { | 830 } else if (command is JSCommandlineCommand) { |
| 877 return new JsCommandlineOutputImpl( | 831 return new JSCommandLineOutput( |
| 878 command, exitCode, timedOut, stdout, stderr, time); | 832 command, exitCode, timedOut, stdout, stderr, time); |
| 879 } else if (command is PubCommand) { | 833 } else if (command is PubCommand) { |
| 880 return new PubCommandOutputImpl( | 834 return new PubCommandOutput( |
| 881 command, exitCode, timedOut, stdout, stderr, time); | 835 command, exitCode, timedOut, stdout, stderr, time); |
| 882 } | 836 } |
| 883 | 837 |
| 884 return new CommandOutputImpl(command, exitCode, timedOut, stdout, stderr, | 838 return new CommandOutput(command, exitCode, timedOut, stdout, stderr, time, |
| 885 time, compilationSkipped, pid); | 839 compilationSkipped, pid); |
| 886 } | 840 } |
| 887 | 841 |
| 888 class UnittestSuiteMessagesMixin { | 842 class UnittestSuiteMessagesMixin { |
| 889 bool _isAsyncTest(String testOutput) { | 843 bool _isAsyncTest(String testOutput) { |
| 890 return testOutput.contains("unittest-suite-wait-for-done"); | 844 return testOutput.contains("unittest-suite-wait-for-done"); |
| 891 } | 845 } |
| 892 | 846 |
| 893 bool _isAsyncTestSuccessful(String testOutput) { | 847 bool _isAsyncTestSuccessful(String testOutput) { |
| 894 return testOutput.contains("unittest-suite-success"); | 848 return testOutput.contains("unittest-suite-success"); |
| 895 } | 849 } |
| 896 | 850 |
| 897 Expectation _negateOutcomeIfIncompleteAsyncTest( | 851 Expectation _negateOutcomeIfIncompleteAsyncTest( |
| 898 Expectation outcome, String testOutput) { | 852 Expectation outcome, String testOutput) { |
| 899 // If this is an asynchronous test and the asynchronous operation didn't | 853 // If this is an asynchronous test and the asynchronous operation didn't |
| 900 // complete successfully, it's outcome is Expectation.FAIL. | 854 // complete successfully, it's outcome is Expectation.FAIL. |
| 901 // TODO: maybe we should introduce a AsyncIncomplete marker or so | 855 // TODO: maybe we should introduce a AsyncIncomplete marker or so |
| 902 if (outcome == Expectation.pass) { | 856 if (outcome == Expectation.pass) { |
| 903 if (_isAsyncTest(testOutput) && !_isAsyncTestSuccessful(testOutput)) { | 857 if (_isAsyncTest(testOutput) && !_isAsyncTestSuccessful(testOutput)) { |
| 904 return Expectation.fail; | 858 return Expectation.fail; |
| 905 } | 859 } |
| 906 } | 860 } |
| 907 return outcome; | 861 return outcome; |
| 908 } | 862 } |
| 909 } | 863 } |
| OLD | NEW |