OLD | NEW |
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 import 'dart:convert'; | 5 import 'dart:convert'; |
6 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow | 6 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow |
7 // CommandOutput.exitCode in subclasses of CommandOutput. | 7 // CommandOutput.exitCode in subclasses of CommandOutput. |
8 import 'dart:io' as io; | 8 import 'dart:io' as io; |
9 | 9 |
10 import 'browser_controller.dart'; | 10 import 'browser_controller.dart'; |
11 import 'command.dart'; | 11 import 'command.dart'; |
12 import 'configuration.dart'; | 12 import 'configuration.dart'; |
13 import 'expectation.dart'; | 13 import 'expectation.dart'; |
14 import 'test_runner.dart'; | 14 import 'test_runner.dart'; |
15 import 'utils.dart'; | 15 import 'utils.dart'; |
16 | 16 |
17 /** | 17 /// CommandOutput records the output of a completed command: the process's exit |
18 * CommandOutput records the output of a completed command: the process's exit | 18 /// code, the standard output and standard error, whether the process timed out, |
19 * code, the standard output and standard error, whether the process timed out, | 19 /// and the time the process took to run. It does not contain a pointer to the |
20 * and the time the process took to run. It does not contain a pointer to the | 20 /// [TestCase] this is the output of, so some functions require the test case |
21 * [TestCase] this is the output of, so some functions require the test case | 21 /// to be passed as an argument. |
22 * to be passed as an argument. | 22 class CommandOutput extends UniqueObject { |
23 */ | 23 final Command command; |
24 abstract class CommandOutput { | |
25 Command get command; | |
26 | 24 |
27 Expectation result(TestCase testCase); | 25 final bool hasTimedOut; |
28 | 26 |
29 bool get hasCrashed; | 27 final Duration time; |
30 | 28 |
31 bool get hasTimedOut; | 29 final int exitCode; |
32 | 30 |
33 bool didFail(TestCase testCase); | 31 final int pid; |
34 | 32 |
35 bool hasFailed(TestCase testCase); | 33 final List<int> stdout; |
| 34 final List<int> stderr; |
36 | 35 |
37 bool get canRunDependendCommands; | 36 final bool compilationSkipped; |
38 | 37 |
39 bool get successful; // otherwise we might to retry running | 38 final List<String> diagnostics = []; |
40 | 39 |
41 Duration get time; | 40 CommandOutput(this.command, this.exitCode, this.hasTimedOut, this.stdout, |
42 | 41 this.stderr, this.time, this.compilationSkipped, this.pid); |
43 int get exitCode; | |
44 | |
45 int get pid; | |
46 | |
47 List<int> get stdout; | |
48 | |
49 List<int> get stderr; | |
50 | |
51 List<String> get diagnostics; | |
52 | |
53 bool get compilationSkipped; | |
54 } | |
55 | |
56 class CommandOutputImpl extends UniqueObject implements CommandOutput { | |
57 Command command; | |
58 int exitCode; | |
59 | |
60 bool timedOut; | |
61 List<int> stdout; | |
62 List<int> stderr; | |
63 Duration time; | |
64 List<String> diagnostics; | |
65 bool compilationSkipped; | |
66 int pid; | |
67 | |
68 /** | |
69 * A flag to indicate we have already printed a warning about ignoring the VM | |
70 * crash, to limit the amount of output produced per test. | |
71 */ | |
72 bool alreadyPrintedWarning = false; | |
73 | |
74 CommandOutputImpl( | |
75 Command this.command, | |
76 int this.exitCode, | |
77 bool this.timedOut, | |
78 List<int> this.stdout, | |
79 List<int> this.stderr, | |
80 Duration this.time, | |
81 bool this.compilationSkipped, | |
82 int this.pid) { | |
83 diagnostics = []; | |
84 } | |
85 | 42 |
86 Expectation result(TestCase testCase) { | 43 Expectation result(TestCase testCase) { |
87 if (hasCrashed) return Expectation.crash; | 44 if (hasCrashed) return Expectation.crash; |
88 if (hasTimedOut) return Expectation.timeout; | 45 if (hasTimedOut) return Expectation.timeout; |
89 if (hasFailed(testCase)) return Expectation.fail; | 46 if (hasFailed(testCase)) return Expectation.fail; |
90 if (hasNonUtf8) return Expectation.nonUtf8Error; | 47 if (hasNonUtf8) return Expectation.nonUtf8Error; |
| 48 |
91 return Expectation.pass; | 49 return Expectation.pass; |
92 } | 50 } |
93 | 51 |
94 bool get hasCrashed { | 52 bool get hasCrashed { |
95 // dart2js exits with code 253 in case of unhandled exceptions. | 53 // dart2js exits with code 253 in case of unhandled exceptions. |
96 // The dart binary exits with code 253 in case of an API error such | 54 // The dart binary exits with code 253 in case of an API error such |
97 // as an invalid snapshot file. | 55 // as an invalid snapshot file. |
98 // In either case an exit code of 253 is considered a crash. | 56 // In either case an exit code of 253 is considered a crash. |
99 if (exitCode == 253) return true; | 57 if (exitCode == 253) return true; |
100 if (io.Platform.operatingSystem == 'windows') { | 58 if (io.Platform.operatingSystem == 'windows') { |
101 // The VM uses std::abort to terminate on asserts. | 59 // The VM uses std::abort to terminate on asserts. |
102 // std::abort terminates with exit code 3 on Windows. | 60 // std::abort terminates with exit code 3 on Windows. |
103 if (exitCode == 3 || exitCode == CRASHING_BROWSER_EXITCODE) { | 61 if (exitCode == 3 || exitCode == browserCrashExitCode) { |
104 return !timedOut; | 62 return !hasTimedOut; |
105 } | 63 } |
106 // If a program receives an uncaught system exception, the program | 64 // If a program receives an uncaught system exception, the program |
107 // terminates with the exception code as exit code. | 65 // terminates with the exception code as exit code. |
108 // The 0x3FFFFF00 mask here tries to determine if an exception indicates | 66 // The 0x3FFFFF00 mask here tries to determine if an exception indicates |
109 // a crash of the program. | 67 // a crash of the program. |
110 // System exception codes can be found in 'winnt.h', for example | 68 // System exception codes can be found in 'winnt.h', for example |
111 // "#define STATUS_ACCESS_VIOLATION ((DWORD) 0xC0000005)" | 69 // "#define STATUS_ACCESS_VIOLATION ((DWORD) 0xC0000005)" |
112 return (!timedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0)); | 70 return (!hasTimedOut && (exitCode < 0) && ((0x3FFFFF00 & exitCode) == 0)); |
113 } | 71 } |
114 return !timedOut && ((exitCode < 0)); | 72 return !hasTimedOut && ((exitCode < 0)); |
115 } | 73 } |
116 | 74 |
117 bool get hasTimedOut => timedOut; | 75 bool _didFail(TestCase testCase) => exitCode != 0 && !hasCrashed; |
118 | |
119 bool didFail(TestCase testCase) { | |
120 return (exitCode != 0 && !hasCrashed); | |
121 } | |
122 | 76 |
123 bool get canRunDependendCommands { | 77 bool get canRunDependendCommands { |
124 // FIXME(kustermann): We may need to change this | 78 // FIXME(kustermann): We may need to change this |
125 return !hasTimedOut && exitCode == 0; | 79 return !hasTimedOut && exitCode == 0; |
126 } | 80 } |
127 | 81 |
128 bool get successful { | 82 bool get successful { |
129 // FIXME(kustermann): We may need to change this | 83 // FIXME(kustermann): We may need to change this |
130 return !hasTimedOut && exitCode == 0; | 84 return !hasTimedOut && exitCode == 0; |
131 } | 85 } |
132 | 86 |
| 87 // TODO(bob): Remove. |
133 // Reverse result of a negative test. | 88 // Reverse result of a negative test. |
134 bool hasFailed(TestCase testCase) { | 89 bool hasFailed(TestCase testCase) { |
135 return testCase.isNegative ? !didFail(testCase) : didFail(testCase); | 90 return testCase.isNegative ? !_didFail(testCase) : _didFail(testCase); |
136 } | 91 } |
137 | 92 |
138 bool get hasNonUtf8 => exitCode == NON_UTF_FAKE_EXITCODE; | 93 bool get hasNonUtf8 => exitCode == nonUtfFakeExitCode; |
139 | 94 |
140 Expectation _negateOutcomeIfNegativeTest( | 95 Expectation _negateOutcomeIfNegativeTest( |
141 Expectation outcome, bool isNegative) { | 96 Expectation outcome, bool isNegative) { |
142 if (!isNegative) return outcome; | 97 if (!isNegative) return outcome; |
143 if (outcome == Expectation.ignore) return outcome; | 98 if (outcome == Expectation.ignore) return outcome; |
144 if (outcome.canBeOutcomeOf(Expectation.fail)) { | 99 if (outcome.canBeOutcomeOf(Expectation.fail)) { |
145 return Expectation.pass; | 100 return Expectation.pass; |
146 } | 101 } |
147 return Expectation.fail; | 102 return Expectation.fail; |
148 } | 103 } |
149 } | 104 } |
150 | 105 |
151 class ContentShellCommandOutputImpl extends CommandOutputImpl { | 106 class ContentShellCommandOutput extends CommandOutput { |
152 // Although tests are reported as passing, content shell sometimes exits with | 107 // Although tests are reported as passing, content shell sometimes exits with |
153 // a nonzero exitcode which makes our dartium builders extremely falky. | 108 // a nonzero exitcode which makes our dartium builders extremely falky. |
154 // See: http://dartbug.com/15139. | 109 // See: http://dartbug.com/15139. |
155 // TODO(rnystrom): Is this still needed? The underlying bug is closed. | 110 // TODO(rnystrom): Is this still needed? The underlying bug is closed. |
156 static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022; | 111 static const _whitelistedContentShellExitCode = -1073740022; |
157 static bool isWindows = io.Platform.operatingSystem == 'windows'; | 112 |
158 static bool _failedBecauseOfFlakyInfrastructure( | 113 static bool _failedBecauseOfFlakyInfrastructure( |
159 Command command, bool timedOut, List<int> stderrBytes) { | 114 Command command, bool timedOut, List<int> stderrBytes) { |
160 // If the browser test failed, it may have been because content shell | 115 // If the browser test failed, it may have been because content shell |
161 // and the virtual framebuffer X server didn't hook up, or it crashed with | 116 // and the virtual framebuffer X server didn't hook up, or it crashed with |
162 // a core dump. Sometimes content shell crashes after it has set the stdout | 117 // a core dump. Sometimes content shell crashes after it has set the stdout |
163 // to PASS, so we have to do this check first. | 118 // to PASS, so we have to do this check first. |
164 // Content shell also fails with a broken pipe message: Issue 26739 | 119 // Content shell also fails with a broken pipe message: Issue 26739 |
165 var zygoteCrash = | 120 var zygoteCrash = |
166 new RegExp(r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe"); | 121 new RegExp(r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe"); |
167 var stderr = decodeUtf8(stderrBytes); | 122 var stderr = decodeUtf8(stderrBytes); |
168 // TODO(7564): See http://dartbug.com/7564 | 123 // TODO(7564): See http://dartbug.com/7564 |
169 // This may not be happening anymore. Test by removing this suppression. | 124 // This may not be happening anymore. Test by removing this suppression. |
170 if (stderr.contains(MESSAGE_CANNOT_OPEN_DISPLAY) || | 125 if (stderr.contains(cannotOpenDisplayMessage) || |
171 stderr.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) { | 126 stderr.contains(failedToRunCommandMessage)) { |
172 DebugLogger.warning( | 127 DebugLogger.warning( |
173 "Warning: Failure because of missing XDisplay. Test ignored"); | 128 "Warning: Failure because of missing XDisplay. Test ignored"); |
174 return true; | 129 return true; |
175 } | 130 } |
176 // TODO(26739): See http://dartbug.com/26739 | 131 // TODO(26739): See http://dartbug.com/26739 |
177 if (zygoteCrash.hasMatch(stderr)) { | 132 if (zygoteCrash.hasMatch(stderr)) { |
178 DebugLogger.warning("Warning: Failure because of content_shell " | 133 DebugLogger.warning("Warning: Failure because of content_shell " |
179 "zygote crash. Test ignored"); | 134 "zygote crash. Test ignored"); |
180 return true; | 135 return true; |
181 } | 136 } |
182 return false; | 137 return false; |
183 } | 138 } |
184 | 139 |
185 bool _infraFailure; | 140 final bool _infraFailure; |
186 | 141 |
187 ContentShellCommandOutputImpl( | 142 ContentShellCommandOutput( |
188 Command command, | 143 Command command, |
189 int exitCode, | 144 int exitCode, |
190 bool timedOut, | 145 bool timedOut, |
191 List<int> stdout, | 146 List<int> stdout, |
192 List<int> stderr, | 147 List<int> stderr, |
193 Duration time, | 148 Duration time, |
194 bool compilationSkipped) | 149 bool compilationSkipped) |
195 : _infraFailure = | 150 : _infraFailure = |
196 _failedBecauseOfFlakyInfrastructure(command, timedOut, stderr), | 151 _failedBecauseOfFlakyInfrastructure(command, timedOut, stderr), |
197 super(command, exitCode, timedOut, stdout, stderr, time, | 152 super(command, exitCode, timedOut, stdout, stderr, time, |
(...skipping 21 matching lines...) Expand all Loading... |
219 return Expectation.fail; | 174 return Expectation.fail; |
220 } | 175 } |
221 return outcome; | 176 return outcome; |
222 } | 177 } |
223 | 178 |
224 bool get successful => canRunDependendCommands; | 179 bool get successful => canRunDependendCommands; |
225 | 180 |
226 bool get canRunDependendCommands { | 181 bool get canRunDependendCommands { |
227 // We cannot rely on the exit code of content_shell as a method to | 182 // We cannot rely on the exit code of content_shell as a method to |
228 // determine if we were successful or not. | 183 // determine if we were successful or not. |
229 return super.canRunDependendCommands && !didFail(null); | 184 return super.canRunDependendCommands && !_didFail(null); |
230 } | 185 } |
231 | 186 |
232 bool get hasCrashed { | 187 bool get hasCrashed => super.hasCrashed || _rendererCrashed; |
233 return super.hasCrashed || _rendererCrashed; | |
234 } | |
235 | 188 |
236 Expectation _getOutcome() { | 189 Expectation _getOutcome() { |
237 if (_browserTestFailure) { | 190 if (_browserTestFailure) { |
238 return Expectation.runtimeError; | 191 return Expectation.runtimeError; |
239 } | 192 } |
240 return Expectation.pass; | 193 return Expectation.pass; |
241 } | 194 } |
242 | 195 |
243 bool get _rendererCrashed => | 196 bool get _rendererCrashed => |
244 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); | 197 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); |
245 | 198 |
246 bool get _browserTestFailure { | 199 bool get _browserTestFailure { |
247 // Browser tests fail unless stdout contains | 200 // Browser tests fail unless stdout contains |
248 // 'Content-Type: text/plain' followed by 'PASS'. | 201 // 'Content-Type: text/plain' followed by 'PASS'. |
249 bool hasContentType = false; | 202 var hasContentType = false; |
250 var stdoutLines = decodeUtf8(super.stdout).split("\n"); | 203 var stdoutLines = decodeUtf8(super.stdout).split("\n"); |
251 var containsFail = false; | 204 var containsFail = false; |
252 var containsPass = false; | 205 var containsPass = false; |
253 for (String line in stdoutLines) { | 206 for (String line in stdoutLines) { |
254 switch (line) { | 207 switch (line) { |
255 case 'Content-Type: text/plain': | 208 case 'Content-Type: text/plain': |
256 hasContentType = true; | 209 hasContentType = true; |
257 break; | 210 break; |
258 case 'FAIL': | 211 case 'FAIL': |
259 if (hasContentType) { | 212 if (hasContentType) { |
(...skipping 20 matching lines...) Expand all Loading... |
280 return true; | 233 return true; |
281 } | 234 } |
282 assert(containsPass); | 235 assert(containsPass); |
283 if (exitCode != 0) { | 236 if (exitCode != 0) { |
284 var message = "All tests passed, but exitCode != 0. " | 237 var message = "All tests passed, but exitCode != 0. " |
285 "Actual exitcode: $exitCode. " | 238 "Actual exitcode: $exitCode. " |
286 "($command)"; | 239 "($command)"; |
287 DebugLogger.warning(message); | 240 DebugLogger.warning(message); |
288 diagnostics.add(message); | 241 diagnostics.add(message); |
289 } | 242 } |
| 243 |
| 244 var isWindows = io.Platform.operatingSystem == 'windows'; |
290 return (!hasCrashed && | 245 return (!hasCrashed && |
291 exitCode != 0 && | 246 exitCode != 0 && |
292 (!isWindows || exitCode != WHITELISTED_CONTENTSHELL_EXITCODE)); | 247 (!isWindows || exitCode != _whitelistedContentShellExitCode)); |
293 } | 248 } |
| 249 |
294 DebugLogger.warning("Couldn't find 'Content-Type: text/plain' in output. " | 250 DebugLogger.warning("Couldn't find 'Content-Type: text/plain' in output. " |
295 "($command)."); | 251 "($command)."); |
296 return true; | 252 return true; |
297 } | 253 } |
298 } | 254 } |
299 | 255 |
300 class HTMLBrowserCommandOutputImpl extends ContentShellCommandOutputImpl { | 256 class HtmlBrowserCommandOutput extends ContentShellCommandOutput { |
301 HTMLBrowserCommandOutputImpl( | 257 HtmlBrowserCommandOutput( |
302 Command command, | 258 Command command, |
303 int exitCode, | 259 int exitCode, |
304 bool timedOut, | 260 bool timedOut, |
305 List<int> stdout, | 261 List<int> stdout, |
306 List<int> stderr, | 262 List<int> stderr, |
307 Duration time, | 263 Duration time, |
308 bool compilationSkipped) | 264 bool compilationSkipped) |
309 : super(command, exitCode, timedOut, stdout, stderr, time, | 265 : super(command, exitCode, timedOut, stdout, stderr, time, |
310 compilationSkipped); | 266 compilationSkipped); |
311 | 267 |
312 bool didFail(TestCase testCase) { | 268 bool _didFail(TestCase testCase) { |
313 return _getOutcome() != Expectation.pass; | 269 return _getOutcome() != Expectation.pass; |
314 } | 270 } |
315 | 271 |
316 bool get _browserTestFailure { | 272 bool get _browserTestFailure { |
317 // We should not need to convert back and forward. | 273 // We should not need to convert back and forward. |
318 var output = decodeUtf8(super.stdout); | 274 var output = decodeUtf8(super.stdout); |
319 if (output.contains("FAIL")) return true; | 275 if (output.contains("FAIL")) return true; |
320 return !output.contains("PASS"); | 276 return !output.contains("PASS"); |
321 } | 277 } |
322 } | 278 } |
323 | 279 |
324 class BrowserTestJsonResult { | 280 class BrowserTestJsonResult { |
325 static const ALLOWED_TYPES = const [ | 281 static const _allowedTypes = const [ |
326 'sync_exception', | 282 'sync_exception', |
327 'window_onerror', | 283 'window_onerror', |
328 'script_onerror', | 284 'script_onerror', |
329 'window_compilationerror', | 285 'window_compilationerror', |
330 'print', | 286 'print', |
331 'message_received', | 287 'message_received', |
332 'dom', | 288 'dom', |
333 'debug' | 289 'debug' |
334 ]; | 290 ]; |
335 | 291 |
(...skipping 10 matching lines...) Expand all Loading... |
346 "$content"; | 302 "$content"; |
347 } | 303 } |
348 } | 304 } |
349 | 305 |
350 try { | 306 try { |
351 var events = JSON.decode(content); | 307 var events = JSON.decode(content); |
352 if (events != null) { | 308 if (events != null) { |
353 validate("Message must be a List", events is List); | 309 validate("Message must be a List", events is List); |
354 | 310 |
355 var messagesByType = <String, List<String>>{}; | 311 var messagesByType = <String, List<String>>{}; |
356 ALLOWED_TYPES.forEach((type) => messagesByType[type] = <String>[]); | 312 _allowedTypes.forEach((type) => messagesByType[type] = <String>[]); |
357 | 313 |
358 for (var entry in events) { | 314 for (var entry in events) { |
359 validate("An entry must be a Map", entry is Map); | 315 validate("An entry must be a Map", entry is Map); |
360 | 316 |
361 var type = entry['type']; | 317 var type = entry['type']; |
362 var value = entry['value'] as String; | 318 var value = entry['value'] as String; |
363 var timestamp = entry['timestamp']; | 319 var timestamp = entry['timestamp']; |
364 | 320 |
365 validate("'type' of an entry must be a String", type is String); | 321 validate("'type' of an entry must be a String", type is String); |
366 validate("'type' has to be in $ALLOWED_TYPES.", | 322 validate("'type' has to be in $_allowedTypes.", |
367 ALLOWED_TYPES.contains(type)); | 323 _allowedTypes.contains(type)); |
368 validate( | 324 validate( |
369 "'timestamp' of an entry must be a number", timestamp is num); | 325 "'timestamp' of an entry must be a number", timestamp is num); |
370 | 326 |
371 messagesByType[type].add(value); | 327 messagesByType[type].add(value); |
372 } | 328 } |
373 validate("The message must have exactly one 'dom' entry.", | 329 validate("The message must have exactly one 'dom' entry.", |
374 messagesByType['dom'].length == 1); | 330 messagesByType['dom'].length == 1); |
375 | 331 |
376 var dom = messagesByType['dom'][0]; | 332 var dom = messagesByType['dom'][0]; |
377 if (dom.endsWith('\n')) { | 333 if (dom.endsWith('\n')) { |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
436 var mainDone = | 392 var mainDone = |
437 searchForMsg(['print', 'message_received'], 'dart-main-done'); | 393 searchForMsg(['print', 'message_received'], 'dart-main-done'); |
438 | 394 |
439 if (mainStarted && mainDone) { | 395 if (mainStarted && mainDone) { |
440 return Expectation.pass; | 396 return Expectation.pass; |
441 } | 397 } |
442 return Expectation.fail; | 398 return Expectation.fail; |
443 } | 399 } |
444 } | 400 } |
445 | 401 |
446 class BrowserCommandOutputImpl extends CommandOutputImpl | 402 class BrowserCommandOutput extends CommandOutput |
447 with UnittestSuiteMessagesMixin { | 403 with UnittestSuiteMessagesMixin { |
448 BrowserTestOutput _result; | 404 final BrowserTestOutput _result; |
449 Expectation _rawOutcome; | 405 final Expectation _rawOutcome; |
450 | 406 |
451 factory BrowserCommandOutputImpl(Command command, BrowserTestOutput result) { | 407 factory BrowserCommandOutput(Command command, BrowserTestOutput result) { |
452 String indent(String string, int numSpaces) { | 408 String indent(String string, int numSpaces) { |
453 var spaces = new List.filled(numSpaces, ' ').join(''); | 409 var spaces = new List.filled(numSpaces, ' ').join(''); |
454 return string | 410 return string |
455 .replaceAll('\r\n', '\n') | 411 .replaceAll('\r\n', '\n') |
456 .split('\n') | 412 .split('\n') |
457 .map((line) => "$spaces$line") | 413 .map((line) => "$spaces$line") |
458 .join('\n'); | 414 .join('\n'); |
459 } | 415 } |
460 | 416 |
461 String stdout = ""; | 417 String stdout = ""; |
(...skipping 30 matching lines...) Expand all Loading... |
492 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; | 448 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; |
493 } | 449 } |
494 | 450 |
495 stderr = '$stderr\n\n' | 451 stderr = '$stderr\n\n' |
496 'BrowserOutput while running the test (* EXPERIMENTAL *):\n' | 452 'BrowserOutput while running the test (* EXPERIMENTAL *):\n' |
497 'BrowserOutput.stdout:\n' | 453 'BrowserOutput.stdout:\n' |
498 '${indent(result.browserOutput.stdout.toString(), 2)}\n' | 454 '${indent(result.browserOutput.stdout.toString(), 2)}\n' |
499 'BrowserOutput.stderr:\n' | 455 'BrowserOutput.stderr:\n' |
500 '${indent(result.browserOutput.stderr.toString(), 2)}\n' | 456 '${indent(result.browserOutput.stderr.toString(), 2)}\n' |
501 '\n'; | 457 '\n'; |
502 return new BrowserCommandOutputImpl._internal( | 458 return new BrowserCommandOutput._internal( |
503 command, result, outcome, encodeUtf8(stdout), encodeUtf8(stderr)); | 459 command, result, outcome, encodeUtf8(stdout), encodeUtf8(stderr)); |
504 } | 460 } |
505 | 461 |
506 BrowserCommandOutputImpl._internal(Command command, BrowserTestOutput result, | 462 BrowserCommandOutput._internal(Command command, BrowserTestOutput result, |
507 this._rawOutcome, List<int> stdout, List<int> stderr) | 463 this._rawOutcome, List<int> stdout, List<int> stderr) |
508 : super(command, 0, result.didTimeout, stdout, stderr, result.duration, | 464 : _result = result, |
509 false, 0) { | 465 super(command, 0, result.didTimeout, stdout, stderr, result.duration, |
510 _result = result; | 466 false, 0); |
511 } | |
512 | 467 |
513 Expectation result(TestCase testCase) { | 468 Expectation result(TestCase testCase) { |
514 // Handle timeouts first | 469 // Handle timeouts first. |
515 if (_result.didTimeout) { | 470 if (_result.didTimeout) { |
516 if (testCase.configuration.runtime == Runtime.ie11) { | 471 if (testCase.configuration.runtime == Runtime.ie11) { |
517 // TODO(28955): See http://dartbug.com/28955 | 472 // TODO(28955): See http://dartbug.com/28955 |
518 DebugLogger.warning("Timeout of ie11 on test ${testCase.displayName}"); | 473 DebugLogger.warning("Timeout of ie11 on test ${testCase.displayName}"); |
519 return Expectation.ignore; | 474 return Expectation.ignore; |
520 } | 475 } |
521 return Expectation.timeout; | 476 return Expectation.timeout; |
522 } | 477 } |
523 | 478 |
524 if (hasNonUtf8) return Expectation.nonUtf8Error; | 479 if (hasNonUtf8) return Expectation.nonUtf8Error; |
525 | 480 |
526 // Multitests are handled specially | 481 // Multitests are handled specially. |
527 if (testCase.hasRuntimeError) { | 482 if (testCase.hasRuntimeError) { |
528 if (_rawOutcome == Expectation.runtimeError) return Expectation.pass; | 483 if (_rawOutcome == Expectation.runtimeError) return Expectation.pass; |
529 return Expectation.missingRuntimeError; | 484 return Expectation.missingRuntimeError; |
530 } | 485 } |
531 | 486 |
532 return _negateOutcomeIfNegativeTest(_rawOutcome, testCase.isNegative); | 487 return _negateOutcomeIfNegativeTest(_rawOutcome, testCase.isNegative); |
533 } | 488 } |
534 } | 489 } |
535 | 490 |
536 class AnalysisCommandOutputImpl extends CommandOutputImpl { | 491 class AnalysisCommandOutput extends CommandOutput { |
537 // An error line has 8 fields that look like: | 492 // An error line has 8 fields that look like: |
538 // ERROR|COMPILER|MISSING_SOURCE|file:/tmp/t.dart|15|1|24|Missing source. | 493 // ERROR|COMPILER|MISSING_SOURCE|file:/tmp/t.dart|15|1|24|Missing source. |
539 final int ERROR_LEVEL = 0; | 494 static const int _errorLevel = 0; |
540 final int ERROR_TYPE = 1; | 495 static const int _formattedError = 7; |
541 final int FILENAME = 3; | |
542 final int FORMATTED_ERROR = 7; | |
543 | 496 |
544 AnalysisCommandOutputImpl( | 497 AnalysisCommandOutput( |
545 Command command, | 498 Command command, |
546 int exitCode, | 499 int exitCode, |
547 bool timedOut, | 500 bool timedOut, |
548 List<int> stdout, | 501 List<int> stdout, |
549 List<int> stderr, | 502 List<int> stderr, |
550 Duration time, | 503 Duration time, |
551 bool compilationSkipped) | 504 bool compilationSkipped) |
552 : super(command, exitCode, timedOut, stdout, stderr, time, | 505 : super(command, exitCode, timedOut, stdout, stderr, time, |
553 compilationSkipped, 0); | 506 compilationSkipped, 0); |
554 | 507 |
555 Expectation result(TestCase testCase) { | 508 Expectation result(TestCase testCase) { |
556 // TODO(kustermann): If we run the analyzer not in batch mode, make sure | 509 // TODO(kustermann): If we run the analyzer not in batch mode, make sure |
557 // that command.exitCodes matches 2 (errors), 1 (warnings), 0 (no warnings, | 510 // that command.exitCodes matches 2 (errors), 1 (warnings), 0 (no warnings, |
558 // no errors) | 511 // no errors) |
559 | 512 |
560 // Handle crashes and timeouts first | 513 // Handle crashes and timeouts first |
561 if (hasCrashed) return Expectation.crash; | 514 if (hasCrashed) return Expectation.crash; |
562 if (hasTimedOut) return Expectation.timeout; | 515 if (hasTimedOut) return Expectation.timeout; |
563 if (hasNonUtf8) return Expectation.nonUtf8Error; | 516 if (hasNonUtf8) return Expectation.nonUtf8Error; |
564 | 517 |
565 // Get the errors/warnings from the analyzer | 518 // Get the errors/warnings from the analyzer |
566 List<String> errors = []; | 519 var errors = <String>[]; |
567 List<String> warnings = []; | 520 var warnings = <String>[]; |
568 parseAnalyzerOutput(errors, warnings); | 521 parseAnalyzerOutput(errors, warnings); |
569 | 522 |
570 // Handle errors / missing errors | 523 // Handle errors / missing errors |
571 if (testCase.expectCompileError) { | 524 if (testCase.expectCompileError) { |
572 if (errors.length > 0) { | 525 if (errors.isNotEmpty) { |
573 return Expectation.pass; | 526 return Expectation.pass; |
574 } | 527 } |
575 return Expectation.missingCompileTimeError; | 528 return Expectation.missingCompileTimeError; |
576 } | 529 } |
577 if (errors.length > 0) { | 530 if (errors.isNotEmpty) { |
578 return Expectation.compileTimeError; | 531 return Expectation.compileTimeError; |
579 } | 532 } |
580 | 533 |
581 // Handle static warnings / missing static warnings | 534 // Handle static warnings / missing static warnings |
582 if (testCase.hasStaticWarning) { | 535 if (testCase.hasStaticWarning) { |
583 if (warnings.length > 0) { | 536 if (warnings.isNotEmpty) { |
584 return Expectation.pass; | 537 return Expectation.pass; |
585 } | 538 } |
586 return Expectation.missingStaticWarning; | 539 return Expectation.missingStaticWarning; |
587 } | 540 } |
588 if (warnings.length > 0) { | 541 if (warnings.isNotEmpty) { |
589 return Expectation.staticWarning; | 542 return Expectation.staticWarning; |
590 } | 543 } |
591 | 544 |
592 assert(errors.length == 0 && warnings.length == 0); | 545 assert(errors.isEmpty && warnings.isEmpty); |
593 assert(!testCase.hasCompileError && !testCase.hasStaticWarning); | 546 assert(!testCase.hasCompileError && !testCase.hasStaticWarning); |
594 return Expectation.pass; | 547 return Expectation.pass; |
595 } | 548 } |
596 | 549 |
597 void parseAnalyzerOutput(List<String> outErrors, List<String> outWarnings) { | 550 void parseAnalyzerOutput(List<String> outErrors, List<String> outWarnings) { |
598 // Parse a line delimited by the | character using \ as an escape character | 551 // Parse a line delimited by the | character using \ as an escape character |
599 // like: FOO|BAR|FOO\|BAR|FOO\\BAZ as 4 fields: FOO BAR FOO|BAR FOO\BAZ | 552 // like: FOO|BAR|FOO\|BAR|FOO\\BAZ as 4 fields: FOO BAR FOO|BAR FOO\BAZ |
600 List<String> splitMachineError(String line) { | 553 List<String> splitMachineError(String line) { |
601 StringBuffer field = new StringBuffer(); | 554 var field = new StringBuffer(); |
602 List<String> result = []; | 555 var result = <String>[]; |
603 bool escaped = false; | 556 var escaped = false; |
604 for (var i = 0; i < line.length; i++) { | 557 for (var i = 0; i < line.length; i++) { |
605 var c = line[i]; | 558 var c = line[i]; |
606 if (!escaped && c == '\\') { | 559 if (!escaped && c == '\\') { |
607 escaped = true; | 560 escaped = true; |
608 continue; | 561 continue; |
609 } | 562 } |
610 escaped = false; | 563 escaped = false; |
611 if (c == '|') { | 564 if (c == '|') { |
612 result.add(field.toString()); | 565 result.add(field.toString()); |
613 field = new StringBuffer(); | 566 field = new StringBuffer(); |
614 continue; | 567 continue; |
615 } | 568 } |
616 field.write(c); | 569 field.write(c); |
617 } | 570 } |
618 result.add(field.toString()); | 571 result.add(field.toString()); |
619 return result; | 572 return result; |
620 } | 573 } |
621 | 574 |
622 for (String line in decodeUtf8(super.stderr).split("\n")) { | 575 for (String line in decodeUtf8(super.stderr).split("\n")) { |
623 if (line.length == 0) continue; | 576 if (line.isEmpty) continue; |
| 577 |
624 List<String> fields = splitMachineError(line); | 578 List<String> fields = splitMachineError(line); |
625 // We only consider errors/warnings for files of interest. | 579 // We only consider errors/warnings for files of interest. |
626 if (fields.length > FORMATTED_ERROR) { | 580 if (fields.length > _formattedError) { |
627 if (fields[ERROR_LEVEL] == 'ERROR') { | 581 if (fields[_errorLevel] == 'ERROR') { |
628 outErrors.add(fields[FORMATTED_ERROR]); | 582 outErrors.add(fields[_formattedError]); |
629 } else if (fields[ERROR_LEVEL] == 'WARNING') { | 583 } else if (fields[_errorLevel] == 'WARNING') { |
630 outWarnings.add(fields[FORMATTED_ERROR]); | 584 outWarnings.add(fields[_formattedError]); |
631 } | 585 } |
632 // OK to Skip error output that doesn't match the machine format | 586 // OK to Skip error output that doesn't match the machine format. |
633 } | 587 } |
634 } | 588 } |
635 } | 589 } |
636 } | 590 } |
637 | 591 |
638 class VmCommandOutputImpl extends CommandOutputImpl | 592 class VMCommandOutput extends CommandOutput with UnittestSuiteMessagesMixin { |
639 with UnittestSuiteMessagesMixin { | 593 static const _dfeErrorExitCode = 252; |
640 static const DART_VM_EXITCODE_DFE_ERROR = 252; | 594 static const _compileErrorExitCode = 254; |
641 static const DART_VM_EXITCODE_COMPILE_TIME_ERROR = 254; | 595 static const _uncaughtExceptionExitCode = 255; |
642 static const DART_VM_EXITCODE_UNCAUGHT_EXCEPTION = 255; | |
643 | 596 |
644 VmCommandOutputImpl(Command command, int exitCode, bool timedOut, | 597 VMCommandOutput(Command command, int exitCode, bool timedOut, |
645 List<int> stdout, List<int> stderr, Duration time, int pid) | 598 List<int> stdout, List<int> stderr, Duration time, int pid) |
646 : super(command, exitCode, timedOut, stdout, stderr, time, false, pid); | 599 : super(command, exitCode, timedOut, stdout, stderr, time, false, pid); |
647 | 600 |
648 Expectation result(TestCase testCase) { | 601 Expectation result(TestCase testCase) { |
649 // Handle crashes and timeouts first | 602 // Handle crashes and timeouts first. |
650 if (exitCode == DART_VM_EXITCODE_DFE_ERROR) return Expectation.dartkCrash; | 603 if (exitCode == _dfeErrorExitCode) return Expectation.dartkCrash; |
651 if (hasCrashed) return Expectation.crash; | 604 if (hasCrashed) return Expectation.crash; |
652 if (hasTimedOut) return Expectation.timeout; | 605 if (hasTimedOut) return Expectation.timeout; |
653 if (hasNonUtf8) return Expectation.nonUtf8Error; | 606 if (hasNonUtf8) return Expectation.nonUtf8Error; |
654 | 607 |
655 // Multitests are handled specially | 608 // Multitests are handled specially. |
656 if (testCase.expectCompileError) { | 609 if (testCase.expectCompileError) { |
657 if (exitCode == DART_VM_EXITCODE_COMPILE_TIME_ERROR) { | 610 if (exitCode == _compileErrorExitCode) { |
658 return Expectation.pass; | 611 return Expectation.pass; |
659 } | 612 } |
660 return Expectation.missingCompileTimeError; | 613 return Expectation.missingCompileTimeError; |
661 } | 614 } |
| 615 |
662 if (testCase.hasRuntimeError) { | 616 if (testCase.hasRuntimeError) { |
663 // TODO(kustermann): Do we consider a "runtimeError" only an uncaught | 617 // TODO(kustermann): Do we consider a "runtimeError" only an uncaught |
664 // exception or does any nonzero exit code fullfil this requirement? | 618 // exception or does any nonzero exit code fullfil this requirement? |
665 if (exitCode != 0) { | 619 if (exitCode != 0) { |
666 return Expectation.pass; | 620 return Expectation.pass; |
667 } | 621 } |
668 return Expectation.missingRuntimeError; | 622 return Expectation.missingRuntimeError; |
669 } | 623 } |
670 | 624 |
671 // The actual outcome depends on the exitCode | 625 // The actual outcome depends on the exitCode. |
672 Expectation outcome; | 626 var outcome = Expectation.pass; |
673 if (exitCode == DART_VM_EXITCODE_COMPILE_TIME_ERROR) { | 627 if (exitCode == _compileErrorExitCode) { |
674 outcome = Expectation.compileTimeError; | 628 outcome = Expectation.compileTimeError; |
675 } else if (exitCode == DART_VM_EXITCODE_UNCAUGHT_EXCEPTION) { | 629 } else if (exitCode == _uncaughtExceptionExitCode) { |
676 outcome = Expectation.runtimeError; | 630 outcome = Expectation.runtimeError; |
677 } else if (exitCode != 0) { | 631 } else if (exitCode != 0) { |
678 // This is a general fail, in case we get an unknown nonzero exitcode. | 632 // This is a general fail, in case we get an unknown nonzero exitcode. |
679 outcome = Expectation.fail; | 633 outcome = Expectation.fail; |
680 } else { | |
681 outcome = Expectation.pass; | |
682 } | 634 } |
| 635 |
683 outcome = _negateOutcomeIfIncompleteAsyncTest(outcome, decodeUtf8(stdout)); | 636 outcome = _negateOutcomeIfIncompleteAsyncTest(outcome, decodeUtf8(stdout)); |
684 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); | 637 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); |
685 } | 638 } |
686 } | 639 } |
687 | 640 |
688 class CompilationCommandOutputImpl extends CommandOutputImpl { | 641 class CompilationCommandOutput extends CommandOutput { |
689 static const DART2JS_EXITCODE_CRASH = 253; | 642 static const _crashExitCode = 253; |
690 | 643 |
691 CompilationCommandOutputImpl( | 644 CompilationCommandOutput( |
692 Command command, | 645 Command command, |
693 int exitCode, | 646 int exitCode, |
694 bool timedOut, | 647 bool timedOut, |
695 List<int> stdout, | 648 List<int> stdout, |
696 List<int> stderr, | 649 List<int> stderr, |
697 Duration time, | 650 Duration time, |
698 bool compilationSkipped) | 651 bool compilationSkipped) |
699 : super(command, exitCode, timedOut, stdout, stderr, time, | 652 : super(command, exitCode, timedOut, stdout, stderr, time, |
700 compilationSkipped, 0); | 653 compilationSkipped, 0); |
701 | 654 |
702 Expectation result(TestCase testCase) { | 655 Expectation result(TestCase testCase) { |
703 // Handle general crash/timeout detection. | 656 // Handle general crash/timeout detection. |
704 if (hasCrashed) return Expectation.crash; | 657 if (hasCrashed) return Expectation.crash; |
705 if (hasTimedOut) { | 658 if (hasTimedOut) { |
706 bool isWindows = io.Platform.operatingSystem == 'windows'; | 659 var isWindows = io.Platform.operatingSystem == 'windows'; |
707 bool isBrowserTestCase = | 660 var isBrowserTestCase = |
708 testCase.commands.any((command) => command is BrowserTestCommand); | 661 testCase.commands.any((command) => command is BrowserTestCommand); |
709 // TODO(26060) Dart2js batch mode hangs on Windows under heavy load. | 662 // TODO(26060) Dart2js batch mode hangs on Windows under heavy load. |
710 return (isWindows && isBrowserTestCase) | 663 return (isWindows && isBrowserTestCase) |
711 ? Expectation.ignore | 664 ? Expectation.ignore |
712 : Expectation.timeout; | 665 : Expectation.timeout; |
713 } | 666 } |
714 if (hasNonUtf8) return Expectation.nonUtf8Error; | 667 if (hasNonUtf8) return Expectation.nonUtf8Error; |
715 | 668 |
716 // Handle dart2js specific crash detection | 669 // Handle dart2js specific crash detection |
717 if (exitCode == DART2JS_EXITCODE_CRASH || | 670 if (exitCode == _crashExitCode || |
718 exitCode == VmCommandOutputImpl.DART_VM_EXITCODE_COMPILE_TIME_ERROR || | 671 exitCode == VMCommandOutput._compileErrorExitCode || |
719 exitCode == VmCommandOutputImpl.DART_VM_EXITCODE_UNCAUGHT_EXCEPTION) { | 672 exitCode == VMCommandOutput._uncaughtExceptionExitCode) { |
720 return Expectation.crash; | 673 return Expectation.crash; |
721 } | 674 } |
722 | 675 |
723 // Multitests are handled specially | 676 // Multitests are handled specially. |
724 if (testCase.expectCompileError) { | 677 if (testCase.expectCompileError) { |
725 // Nonzero exit code of the compiler means compilation failed | 678 // Nonzero exit code of the compiler means compilation failed |
726 // TODO(kustermann): Do we have a special exit code in that case??? | 679 // TODO(kustermann): Do we have a special exit code in that case??? |
727 if (exitCode != 0) { | 680 if (exitCode != 0) { |
728 return Expectation.pass; | 681 return Expectation.pass; |
729 } | 682 } |
730 return Expectation.missingCompileTimeError; | 683 return Expectation.missingCompileTimeError; |
731 } | 684 } |
732 | 685 |
733 // TODO(kustermann): This is a hack, remove it | 686 // TODO(kustermann): This is a hack, remove it. |
734 if (testCase.hasRuntimeError && testCase.commands.length > 1) { | 687 if (testCase.hasRuntimeError && testCase.commands.length > 1) { |
735 // We expected to run the test, but we got an compile time error. | 688 // We expected to run the test, but we got an compile time error. |
736 // If the compilation succeeded, we wouldn't be in here! | 689 // If the compilation succeeded, we wouldn't be in here! |
737 assert(exitCode != 0); | 690 assert(exitCode != 0); |
738 return Expectation.compileTimeError; | 691 return Expectation.compileTimeError; |
739 } | 692 } |
740 | 693 |
741 Expectation outcome = | 694 var outcome = |
742 exitCode == 0 ? Expectation.pass : Expectation.compileTimeError; | 695 exitCode == 0 ? Expectation.pass : Expectation.compileTimeError; |
743 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); | 696 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); |
744 } | 697 } |
745 } | 698 } |
746 | 699 |
747 class KernelCompilationCommandOutputImpl extends CompilationCommandOutputImpl { | 700 class KernelCompilationCommandOutput extends CompilationCommandOutput { |
748 KernelCompilationCommandOutputImpl( | 701 KernelCompilationCommandOutput( |
749 Command command, | 702 Command command, |
750 int exitCode, | 703 int exitCode, |
751 bool timedOut, | 704 bool timedOut, |
752 List<int> stdout, | 705 List<int> stdout, |
753 List<int> stderr, | 706 List<int> stderr, |
754 Duration time, | 707 Duration time, |
755 bool compilationSkipped) | 708 bool compilationSkipped) |
756 : super(command, exitCode, timedOut, stdout, stderr, time, | 709 : super(command, exitCode, timedOut, stdout, stderr, time, |
757 compilationSkipped); | 710 compilationSkipped); |
758 | 711 |
759 bool get canRunDependendCommands { | 712 bool get canRunDependendCommands { |
760 // See [BatchRunnerProcess]: 0 means success, 1 means compile-time error. | 713 // See [BatchRunnerProcess]: 0 means success, 1 means compile-time error. |
761 // TODO(asgerf): When the frontend supports it, continue running even if | 714 // TODO(asgerf): When the frontend supports it, continue running even if |
762 // there were compile-time errors. See kernel_sdk issue #18. | 715 // there were compile-time errors. See kernel_sdk issue #18. |
763 return !hasCrashed && !timedOut && exitCode == 0; | 716 return !hasCrashed && !hasTimedOut && exitCode == 0; |
764 } | 717 } |
765 | 718 |
766 Expectation result(TestCase testCase) { | 719 Expectation result(TestCase testCase) { |
767 Expectation result = super.result(testCase); | 720 Expectation result = super.result(testCase); |
768 if (result.canBeOutcomeOf(Expectation.crash)) { | 721 if (result.canBeOutcomeOf(Expectation.crash)) { |
769 return Expectation.dartkCrash; | 722 return Expectation.dartkCrash; |
770 } else if (result.canBeOutcomeOf(Expectation.timeout)) { | 723 } else if (result.canBeOutcomeOf(Expectation.timeout)) { |
771 return Expectation.dartkTimeout; | 724 return Expectation.dartkTimeout; |
772 } else if (result.canBeOutcomeOf(Expectation.compileTimeError)) { | 725 } else if (result.canBeOutcomeOf(Expectation.compileTimeError)) { |
773 return Expectation.dartkCompileTimeError; | 726 return Expectation.dartkCompileTimeError; |
774 } | 727 } |
775 return result; | 728 return result; |
776 } | 729 } |
777 | 730 |
778 // If the compiler was able to produce a Kernel IR file we want to run the | 731 /// If the compiler was able to produce a Kernel IR file we want to run the |
779 // result on the Dart VM. We therefore mark the [KernelCompilationCommand] as | 732 /// result on the Dart VM. We therefore mark the [KernelCompilationCommand] |
780 // successful. | 733 /// as successful. |
781 // => This ensures we test that the DartVM produces correct CompileTime errors | 734 /// |
782 // as it is supposed to for our test suites. | 735 /// This ensures we test that the DartVM produces correct CompileTime errors |
| 736 /// as it is supposed to for our test suites. |
783 bool get successful => canRunDependendCommands; | 737 bool get successful => canRunDependendCommands; |
784 } | 738 } |
785 | 739 |
786 class JsCommandlineOutputImpl extends CommandOutputImpl | 740 class JSCommandLineOutput extends CommandOutput |
787 with UnittestSuiteMessagesMixin { | 741 with UnittestSuiteMessagesMixin { |
788 JsCommandlineOutputImpl(Command command, int exitCode, bool timedOut, | 742 JSCommandLineOutput(Command command, int exitCode, bool timedOut, |
789 List<int> stdout, List<int> stderr, Duration time) | 743 List<int> stdout, List<int> stderr, Duration time) |
790 : super(command, exitCode, timedOut, stdout, stderr, time, false, 0); | 744 : super(command, exitCode, timedOut, stdout, stderr, time, false, 0); |
791 | 745 |
792 Expectation result(TestCase testCase) { | 746 Expectation result(TestCase testCase) { |
793 // Handle crashes and timeouts first | 747 // Handle crashes and timeouts first. |
794 if (hasCrashed) return Expectation.crash; | 748 if (hasCrashed) return Expectation.crash; |
795 if (hasTimedOut) return Expectation.timeout; | 749 if (hasTimedOut) return Expectation.timeout; |
796 if (hasNonUtf8) return Expectation.nonUtf8Error; | 750 if (hasNonUtf8) return Expectation.nonUtf8Error; |
797 | 751 |
798 if (testCase.hasRuntimeError) { | 752 if (testCase.hasRuntimeError) { |
799 if (exitCode != 0) return Expectation.pass; | 753 if (exitCode != 0) return Expectation.pass; |
800 return Expectation.missingRuntimeError; | 754 return Expectation.missingRuntimeError; |
801 } | 755 } |
802 | 756 |
803 var outcome = exitCode == 0 ? Expectation.pass : Expectation.runtimeError; | 757 var outcome = exitCode == 0 ? Expectation.pass : Expectation.runtimeError; |
804 outcome = _negateOutcomeIfIncompleteAsyncTest(outcome, decodeUtf8(stdout)); | 758 outcome = _negateOutcomeIfIncompleteAsyncTest(outcome, decodeUtf8(stdout)); |
805 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); | 759 return _negateOutcomeIfNegativeTest(outcome, testCase.isNegative); |
806 } | 760 } |
807 } | 761 } |
808 | 762 |
809 class PubCommandOutputImpl extends CommandOutputImpl { | 763 class PubCommandOutput extends CommandOutput { |
810 PubCommandOutputImpl(PubCommand command, int exitCode, bool timedOut, | 764 PubCommandOutput(PubCommand command, int exitCode, bool timedOut, |
811 List<int> stdout, List<int> stderr, Duration time) | 765 List<int> stdout, List<int> stderr, Duration time) |
812 : super(command, exitCode, timedOut, stdout, stderr, time, false, 0); | 766 : super(command, exitCode, timedOut, stdout, stderr, time, false, 0); |
813 | 767 |
814 Expectation result(TestCase testCase) { | 768 Expectation result(TestCase testCase) { |
815 // Handle crashes and timeouts first | 769 // Handle crashes and timeouts first. |
816 if (hasCrashed) return Expectation.crash; | 770 if (hasCrashed) return Expectation.crash; |
817 if (hasTimedOut) return Expectation.timeout; | 771 if (hasTimedOut) return Expectation.timeout; |
818 if (hasNonUtf8) return Expectation.nonUtf8Error; | 772 if (hasNonUtf8) return Expectation.nonUtf8Error; |
819 | 773 |
820 if (exitCode == 0) { | 774 if (exitCode == 0) { |
821 return Expectation.pass; | 775 return Expectation.pass; |
822 } else if ((command as PubCommand).command == 'get') { | 776 } else if ((command as PubCommand).command == 'get') { |
823 return Expectation.pubGetError; | 777 return Expectation.pubGetError; |
824 } else { | 778 } else { |
825 return Expectation.fail; | 779 return Expectation.fail; |
826 } | 780 } |
827 } | 781 } |
828 } | 782 } |
829 | 783 |
830 class ScriptCommandOutputImpl extends CommandOutputImpl { | 784 class ScriptCommandOutput extends CommandOutput { |
831 final Expectation _result; | 785 final Expectation _result; |
832 | 786 |
833 ScriptCommandOutputImpl(ScriptCommand command, this._result, | 787 ScriptCommandOutput(ScriptCommand command, this._result, |
834 String scriptExecutionInformation, Duration time) | 788 String scriptExecutionInformation, Duration time) |
835 : super(command, 0, false, [], [], time, false, 0) { | 789 : super(command, 0, false, [], [], time, false, 0) { |
836 var lines = scriptExecutionInformation.split("\n"); | 790 var lines = scriptExecutionInformation.split("\n"); |
837 diagnostics.addAll(lines); | 791 diagnostics.addAll(lines); |
838 } | 792 } |
839 | 793 |
840 Expectation result(TestCase testCase) => _result; | 794 Expectation result(TestCase testCase) => _result; |
841 | 795 |
842 bool get canRunDependendCommands => _result == Expectation.pass; | 796 bool get canRunDependendCommands => _result == Expectation.pass; |
843 | 797 |
844 bool get successful => _result == Expectation.pass; | 798 bool get successful => _result == Expectation.pass; |
845 } | 799 } |
846 | 800 |
847 CommandOutput createCommandOutput(Command command, int exitCode, bool timedOut, | 801 CommandOutput createCommandOutput(Command command, int exitCode, bool timedOut, |
848 List<int> stdout, List<int> stderr, Duration time, bool compilationSkipped, | 802 List<int> stdout, List<int> stderr, Duration time, bool compilationSkipped, |
849 [int pid = 0]) { | 803 [int pid = 0]) { |
850 if (command is ContentShellCommand) { | 804 if (command is ContentShellCommand) { |
851 return new ContentShellCommandOutputImpl( | 805 return new ContentShellCommandOutput( |
852 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 806 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
853 } else if (command is BrowserTestCommand) { | 807 } else if (command is BrowserTestCommand) { |
854 return new HTMLBrowserCommandOutputImpl( | 808 return new HtmlBrowserCommandOutput( |
855 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 809 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
856 } else if (command is AnalysisCommand) { | 810 } else if (command is AnalysisCommand) { |
857 return new AnalysisCommandOutputImpl( | 811 return new AnalysisCommandOutput( |
858 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 812 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
859 } else if (command is VmCommand) { | 813 } else if (command is VmCommand) { |
860 return new VmCommandOutputImpl( | 814 return new VMCommandOutput( |
861 command, exitCode, timedOut, stdout, stderr, time, pid); | 815 command, exitCode, timedOut, stdout, stderr, time, pid); |
862 } else if (command is KernelCompilationCommand) { | 816 } else if (command is KernelCompilationCommand) { |
863 return new KernelCompilationCommandOutputImpl( | 817 return new KernelCompilationCommandOutput( |
864 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 818 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
865 } else if (command is AdbPrecompilationCommand) { | 819 } else if (command is AdbPrecompilationCommand) { |
866 return new VmCommandOutputImpl( | 820 return new VMCommandOutput( |
867 command, exitCode, timedOut, stdout, stderr, time, pid); | 821 command, exitCode, timedOut, stdout, stderr, time, pid); |
868 } else if (command is CompilationCommand) { | 822 } else if (command is CompilationCommand) { |
869 if (command.displayName == 'precompiler' || | 823 if (command.displayName == 'precompiler' || |
870 command.displayName == 'app_jit') { | 824 command.displayName == 'app_jit') { |
871 return new VmCommandOutputImpl( | 825 return new VMCommandOutput( |
872 command, exitCode, timedOut, stdout, stderr, time, pid); | 826 command, exitCode, timedOut, stdout, stderr, time, pid); |
873 } | 827 } |
874 return new CompilationCommandOutputImpl( | 828 return new CompilationCommandOutput( |
875 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); | 829 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped); |
876 } else if (command is JSCommandlineCommand) { | 830 } else if (command is JSCommandlineCommand) { |
877 return new JsCommandlineOutputImpl( | 831 return new JSCommandLineOutput( |
878 command, exitCode, timedOut, stdout, stderr, time); | 832 command, exitCode, timedOut, stdout, stderr, time); |
879 } else if (command is PubCommand) { | 833 } else if (command is PubCommand) { |
880 return new PubCommandOutputImpl( | 834 return new PubCommandOutput( |
881 command, exitCode, timedOut, stdout, stderr, time); | 835 command, exitCode, timedOut, stdout, stderr, time); |
882 } | 836 } |
883 | 837 |
884 return new CommandOutputImpl(command, exitCode, timedOut, stdout, stderr, | 838 return new CommandOutput(command, exitCode, timedOut, stdout, stderr, time, |
885 time, compilationSkipped, pid); | 839 compilationSkipped, pid); |
886 } | 840 } |
887 | 841 |
888 class UnittestSuiteMessagesMixin { | 842 class UnittestSuiteMessagesMixin { |
889 bool _isAsyncTest(String testOutput) { | 843 bool _isAsyncTest(String testOutput) { |
890 return testOutput.contains("unittest-suite-wait-for-done"); | 844 return testOutput.contains("unittest-suite-wait-for-done"); |
891 } | 845 } |
892 | 846 |
893 bool _isAsyncTestSuccessful(String testOutput) { | 847 bool _isAsyncTestSuccessful(String testOutput) { |
894 return testOutput.contains("unittest-suite-success"); | 848 return testOutput.contains("unittest-suite-success"); |
895 } | 849 } |
896 | 850 |
897 Expectation _negateOutcomeIfIncompleteAsyncTest( | 851 Expectation _negateOutcomeIfIncompleteAsyncTest( |
898 Expectation outcome, String testOutput) { | 852 Expectation outcome, String testOutput) { |
899 // If this is an asynchronous test and the asynchronous operation didn't | 853 // If this is an asynchronous test and the asynchronous operation didn't |
900 // complete successfully, it's outcome is Expectation.FAIL. | 854 // complete successfully, it's outcome is Expectation.FAIL. |
901 // TODO: maybe we should introduce a AsyncIncomplete marker or so | 855 // TODO: maybe we should introduce a AsyncIncomplete marker or so |
902 if (outcome == Expectation.pass) { | 856 if (outcome == Expectation.pass) { |
903 if (_isAsyncTest(testOutput) && !_isAsyncTestSuccessful(testOutput)) { | 857 if (_isAsyncTest(testOutput) && !_isAsyncTestSuccessful(testOutput)) { |
904 return Expectation.fail; | 858 return Expectation.fail; |
905 } | 859 } |
906 } | 860 } |
907 return outcome; | 861 return outcome; |
908 } | 862 } |
909 } | 863 } |
OLD | NEW |