OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 903 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
914 return Expectation.FAIL; | 914 return Expectation.FAIL; |
915 } | 915 } |
916 } | 916 } |
917 return outcome; | 917 return outcome; |
918 } | 918 } |
919 } | 919 } |
920 | 920 |
921 /** | 921 /** |
922 * CommandOutput records the output of a completed command: the process's exit | 922 * CommandOutput records the output of a completed command: the process's exit |
923 * code, the standard output and standard error, whether the process timed out, | 923 * code, the standard output and standard error, whether the process timed out, |
924 * and the time the process took to run. It also contains a pointer to the | 924 * and the time the process took to run. It does not contain a pointer to the |
925 * [TestCase] this is the output of. | 925 * [TestCase] this is the output of, so some functions require the test case |
| 926 * to be passed as an argument. |
926 */ | 927 */ |
927 abstract class CommandOutput { | 928 abstract class CommandOutput { |
928 Command get command; | 929 Command get command; |
929 | 930 |
930 Expectation result(TestCase testCase); | 931 Expectation result(TestCase testCase); |
931 | 932 |
932 bool get hasCrashed; | 933 bool get hasCrashed; |
933 | 934 |
934 bool get hasTimedOut; | 935 bool get hasTimedOut; |
935 | 936 |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
967 List<String> diagnostics; | 968 List<String> diagnostics; |
968 bool compilationSkipped; | 969 bool compilationSkipped; |
969 int pid; | 970 int pid; |
970 | 971 |
971 /** | 972 /** |
972 * A flag to indicate we have already printed a warning about ignoring the VM | 973 * A flag to indicate we have already printed a warning about ignoring the VM |
973 * crash, to limit the amount of output produced per test. | 974 * crash, to limit the amount of output produced per test. |
974 */ | 975 */ |
975 bool alreadyPrintedWarning = false; | 976 bool alreadyPrintedWarning = false; |
976 | 977 |
977 // TODO(kustermann): Remove testCase from this class. | |
978 CommandOutputImpl( | 978 CommandOutputImpl( |
979 Command this.command, | 979 Command this.command, |
980 int this.exitCode, | 980 int this.exitCode, |
981 bool this.timedOut, | 981 bool this.timedOut, |
982 List<int> this.stdout, | 982 List<int> this.stdout, |
983 List<int> this.stderr, | 983 List<int> this.stderr, |
984 Duration this.time, | 984 Duration this.time, |
985 bool this.compilationSkipped, | 985 bool this.compilationSkipped, |
986 int this.pid) { | 986 int this.pid) { |
987 diagnostics = []; | 987 diagnostics = []; |
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1047 return Expectation.FAIL; | 1047 return Expectation.FAIL; |
1048 } | 1048 } |
1049 } | 1049 } |
1050 | 1050 |
1051 class BrowserCommandOutputImpl extends CommandOutputImpl { | 1051 class BrowserCommandOutputImpl extends CommandOutputImpl { |
1052 // Although tests are reported as passing, content shell sometimes exits with | 1052 // Although tests are reported as passing, content shell sometimes exits with |
1053 // a nonzero exitcode which makes our dartium builders extremely falky. | 1053 // a nonzero exitcode which makes our dartium builders extremely falky. |
1054 // See: http://dartbug.com/15139. | 1054 // See: http://dartbug.com/15139. |
1055 static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022; | 1055 static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022; |
1056 static bool isWindows = io.Platform.operatingSystem == 'windows'; | 1056 static bool isWindows = io.Platform.operatingSystem == 'windows'; |
| 1057 static bool _failedBecauseOfFlakyInfrastructure(List<int> stderrBytes) { |
| 1058 // If the browser test failed, it may have been because content shell |
| 1059 // and the virtual framebuffer X server didn't hook up, or it crashed with |
| 1060 // a core dump. Sometimes content shell crashes after it has set the stdout |
| 1061 // to PASS, so we have to do this check first. |
| 1062 // Content shell also fails with a broken pipe message: Issue 26739 |
| 1063 var zygoteCrash = new RegExp( |
| 1064 r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe"); |
| 1065 var stderr = decodeUtf8(stderrBytes); |
| 1066 // TODO(whesse): Issue: 7564 |
| 1067 // This may not be happening anymore. Test by removing this suppression. |
| 1068 if (stderr.contains(MESSAGE_CANNOT_OPEN_DISPLAY) || |
| 1069 stderr.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) { |
| 1070 DebugLogger.warning( |
| 1071 "Warning: Failure because of missing XDisplay. Test ignored"); |
| 1072 return true; |
| 1073 } |
| 1074 // Issue 26739 |
| 1075 if (zygoteCrash.hasMatch(stderr)) { |
| 1076 DebugLogger.warning("Warning: Failure because of content_shell " |
| 1077 "zygote crash. Test ignored"); |
| 1078 return true; |
| 1079 } |
| 1080 return false; |
| 1081 } |
1057 | 1082 |
1058 bool _failedBecauseOfMissingXDisplay; | 1083 bool _infraFailure; |
1059 | 1084 |
1060 BrowserCommandOutputImpl( | 1085 BrowserCommandOutputImpl( |
1061 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped) | 1086 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped) |
1062 : super(command, exitCode, timedOut, stdout, stderr, time, | 1087 : super(command, exitCode, timedOut, stdout, stderr, time, |
1063 compilationSkipped, 0) { | 1088 compilationSkipped, 0), |
1064 _failedBecauseOfMissingXDisplay = _didFailBecauseOfMissingXDisplay(); | 1089 _infraFailure = _failedBecauseOfFlakyInfrastructure(stderr); |
1065 if (_failedBecauseOfMissingXDisplay) { | |
1066 DebugLogger.warning("Warning: Test failure because of missing XDisplay"); | |
1067 // If we get the X server error, or DRT crashes with a core dump, retry | |
1068 // the test. | |
1069 } | |
1070 } | |
1071 | 1090 |
1072 Expectation result(TestCase testCase) { | 1091 Expectation result(TestCase testCase) { |
1073 // Handle crashes and timeouts first | 1092 // Handle crashes and timeouts first |
1074 if (hasCrashed) return Expectation.CRASH; | 1093 if (hasCrashed) return Expectation.CRASH; |
1075 if (hasTimedOut) return Expectation.TIMEOUT; | 1094 if (hasTimedOut) return Expectation.TIMEOUT; |
1076 | 1095 |
| 1096 if (_infraFailure) { |
| 1097 return Expectation.IGNORE; |
| 1098 } |
1077 var outcome = _getOutcome(); | 1099 var outcome = _getOutcome(); |
1078 | 1100 |
1079 if (testCase.hasRuntimeError) { | 1101 if (testCase.hasRuntimeError) { |
1080 if (!outcome.canBeOutcomeOf(Expectation.RUNTIME_ERROR)) { | 1102 if (!outcome.canBeOutcomeOf(Expectation.RUNTIME_ERROR)) { |
1081 return Expectation.MISSING_RUNTIME_ERROR; | 1103 return Expectation.MISSING_RUNTIME_ERROR; |
1082 } | 1104 } |
1083 } | 1105 } |
1084 if (testCase.isNegative) { | 1106 if (testCase.isNegative) { |
1085 if (outcome.canBeOutcomeOf(Expectation.FAIL)) return Expectation.PASS; | 1107 if (outcome.canBeOutcomeOf(Expectation.FAIL)) return Expectation.PASS; |
1086 return Expectation.FAIL; | 1108 return Expectation.FAIL; |
1087 } | 1109 } |
1088 return outcome; | 1110 return outcome; |
1089 } | 1111 } |
1090 | 1112 |
1091 bool get successful => canRunDependendCommands; | 1113 bool get successful => canRunDependendCommands; |
1092 | 1114 |
1093 bool get canRunDependendCommands { | 1115 bool get canRunDependendCommands { |
1094 // We cannot rely on the exit code of content_shell as a method to determine | 1116 // We cannot rely on the exit code of content_shell as a method to |
1095 // if we were successful or not. | 1117 // determine if we were successful or not. |
1096 return super.canRunDependendCommands && !didFail(null); | 1118 return super.canRunDependendCommands && !didFail(null); |
1097 } | 1119 } |
1098 | 1120 |
1099 bool get hasCrashed { | 1121 bool get hasCrashed { |
1100 return super.hasCrashed || _rendererCrashed; | 1122 return super.hasCrashed || _rendererCrashed; |
1101 } | 1123 } |
1102 | 1124 |
1103 Expectation _getOutcome() { | 1125 Expectation _getOutcome() { |
1104 if (_failedBecauseOfMissingXDisplay) { | |
1105 return Expectation.FAIL; | |
1106 } | |
1107 | |
1108 if (_browserTestFailure) { | 1126 if (_browserTestFailure) { |
1109 return Expectation.RUNTIME_ERROR; | 1127 return Expectation.RUNTIME_ERROR; |
1110 } | 1128 } |
1111 return Expectation.PASS; | 1129 return Expectation.PASS; |
1112 } | 1130 } |
1113 | 1131 |
1114 bool _didFailBecauseOfMissingXDisplay() { | |
1115 // Browser case: | |
1116 // If the browser test failed, it may have been because content shell | |
1117 // and the virtual framebuffer X server didn't hook up, or it crashed with | |
1118 // a core dump. Sometimes content shell crashes after it has set the stdout | |
1119 // to PASS, so we have to do this check first. | |
1120 var stderrLines = decodeUtf8(super.stderr).split("\n"); | |
1121 for (String line in stderrLines) { | |
1122 // TODO(kustermann,ricow): Issue: 7564 | |
1123 // This seems to happen quite frequently, we need to figure out why. | |
1124 if (line.contains(MESSAGE_CANNOT_OPEN_DISPLAY) || | |
1125 line.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) { | |
1126 return true; | |
1127 } | |
1128 } | |
1129 return false; | |
1130 } | |
1131 | |
1132 bool get _rendererCrashed => | 1132 bool get _rendererCrashed => |
1133 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); | 1133 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); |
1134 | 1134 |
1135 bool get _browserTestFailure { | 1135 bool get _browserTestFailure { |
1136 // Browser tests fail unless stdout contains | 1136 // Browser tests fail unless stdout contains |
1137 // 'Content-Type: text/plain' followed by 'PASS'. | 1137 // 'Content-Type: text/plain' followed by 'PASS'. |
1138 bool hasContentType = false; | 1138 bool hasContentType = false; |
1139 var stdoutLines = decodeUtf8(super.stdout).split("\n"); | 1139 var stdoutLines = decodeUtf8(super.stdout).split("\n"); |
1140 var containsFail = false; | 1140 var containsFail = false; |
1141 var containsPass = false; | 1141 var containsPass = false; |
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1327 } | 1327 } |
1328 } | 1328 } |
1329 | 1329 |
1330 class BrowserControllerTestOutcome extends CommandOutputImpl | 1330 class BrowserControllerTestOutcome extends CommandOutputImpl |
1331 with UnittestSuiteMessagesMixin { | 1331 with UnittestSuiteMessagesMixin { |
1332 BrowserTestOutput _result; | 1332 BrowserTestOutput _result; |
1333 Expectation _rawOutcome; | 1333 Expectation _rawOutcome; |
1334 | 1334 |
1335 factory BrowserControllerTestOutcome( | 1335 factory BrowserControllerTestOutcome( |
1336 Command command, BrowserTestOutput result) { | 1336 Command command, BrowserTestOutput result) { |
1337 void validate(String assertion, bool value) { | |
1338 if (!value) { | |
1339 throw "InvalidFormat sent from browser driving page: $assertion:\n\n" | |
1340 "${result.lastKnownMessage}"; | |
1341 } | |
1342 } | |
1343 | |
1344 String indent(String string, int numSpaces) { | 1337 String indent(String string, int numSpaces) { |
1345 var spaces = new List.filled(numSpaces, ' ').join(''); | 1338 var spaces = new List.filled(numSpaces, ' ').join(''); |
1346 return string | 1339 return string |
1347 .replaceAll('\r\n', '\n') | 1340 .replaceAll('\r\n', '\n') |
1348 .split('\n') | 1341 .split('\n') |
1349 .map((line) => "$spaces$line") | 1342 .map((line) => "$spaces$line") |
1350 .join('\n'); | 1343 .join('\n'); |
1351 } | 1344 } |
1352 | 1345 |
1353 String stdout = ""; | 1346 String stdout = ""; |
(...skipping 13 matching lines...) Expand all Loading... |
1367 } else { | 1360 } else { |
1368 outcome = Expectation.RUNTIME_ERROR; | 1361 outcome = Expectation.RUNTIME_ERROR; |
1369 } | 1362 } |
1370 } | 1363 } |
1371 | 1364 |
1372 if (result.didTimeout) { | 1365 if (result.didTimeout) { |
1373 if (result.delayUntilTestStarted != null) { | 1366 if (result.delayUntilTestStarted != null) { |
1374 stderr = "This test timed out. The delay until the test actually " | 1367 stderr = "This test timed out. The delay until the test actually " |
1375 "started was: ${result.delayUntilTestStarted}."; | 1368 "started was: ${result.delayUntilTestStarted}."; |
1376 } else { | 1369 } else { |
1377 // TODO(ricow/kustermann) as soon as we record the state periodically, | 1370 stderr = "This test has not notified test.py that it started running."; |
1378 // we will have more information and can remove this warning. | |
1379 stderr = "This test has not notified test.py that it started running. " | |
1380 "This could be a bug in test.py! " | |
1381 "Please contact ricow/whesse"; | |
1382 } | 1371 } |
1383 } | 1372 } |
1384 | 1373 |
1385 if (parsedResult != null) { | 1374 if (parsedResult != null) { |
1386 stdout = "events:\n${indent(prettifyJson(parsedResult.events), 2)}\n\n"; | 1375 stdout = "events:\n${indent(prettifyJson(parsedResult.events), 2)}\n\n"; |
1387 } else { | 1376 } else { |
1388 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; | 1377 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; |
1389 } | 1378 } |
1390 | 1379 |
1391 stderr = '$stderr\n\n' | 1380 stderr = '$stderr\n\n' |
(...skipping 1659 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3051 } | 3040 } |
3052 } | 3041 } |
3053 | 3042 |
3054 void eventAllTestsDone() { | 3043 void eventAllTestsDone() { |
3055 for (var listener in _eventListener) { | 3044 for (var listener in _eventListener) { |
3056 listener.allDone(); | 3045 listener.allDone(); |
3057 } | 3046 } |
3058 _allDone(); | 3047 _allDone(); |
3059 } | 3048 } |
3060 } | 3049 } |
OLD | NEW |