OLD | NEW |
---|---|
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
(...skipping 1036 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1047 return Expectation.FAIL; | 1047 return Expectation.FAIL; |
1048 } | 1048 } |
1049 } | 1049 } |
1050 | 1050 |
1051 class BrowserCommandOutputImpl extends CommandOutputImpl { | 1051 class BrowserCommandOutputImpl extends CommandOutputImpl { |
1052 // Although tests are reported as passing, content shell sometimes exits with | 1052 // Although tests are reported as passing, content shell sometimes exits with |
1053 // a nonzero exitcode which makes our dartium builders extremely falky. | 1053 // a nonzero exitcode which makes our dartium builders extremely falky. |
1054 // See: http://dartbug.com/15139. | 1054 // See: http://dartbug.com/15139. |
1055 static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022; | 1055 static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022; |
1056 static bool isWindows = io.Platform.operatingSystem == 'windows'; | 1056 static bool isWindows = io.Platform.operatingSystem == 'windows'; |
1057 static bool _failedBecauseOfFlakyInfrastructure(List<int> stderr) { | |
1058 // If the browser test failed, it may have been because content shell | |
1059 // and the virtual framebuffer X server didn't hook up, or it crashed with | |
1060 // a core dump. Sometimes content shell crashes after it has set the stdout | |
1061 // to PASS, so we have to do this check first. | |
1062 // Content shell also fails with a broken pipe message: Issue 26739 | |
1063 var zygoteCrash = new RegExp( | |
1064 r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe"); | |
1065 var stderrLines = decodeUtf8(stderr).split("\n"); | |
1066 for (String line in stderrLines) { | |
kustermann
2016/06/21 14:22:19
I think you can avoid the split into lines & itera
Bill Hesse
2016/06/24 12:06:59
Done.
| |
1067 // TODO(whesse): Issue: 7564 | |
1068 // This may not be happening anymore. Test by removing this suppression. | |
1069 if (line.contains(MESSAGE_CANNOT_OPEN_DISPLAY) || | |
1070 line.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) { | |
1071 DebugLogger.warning( | |
1072 "Warning: Failure because of missing XDisplay. Test ignored"); | |
1073 return true; | |
1074 } | |
1075 // Issue 26739 | |
1076 if (zygoteCrash.hasMatch(line)) { | |
1077 DebugLogger.warning("Warning: Failure because of content_shell " | |
1078 "zygote crash. Test ignored"); | |
1079 return true; | |
1080 } | |
1081 } | |
1082 return false; | |
1083 } | |
1057 | 1084 |
1058 bool _failedBecauseOfMissingXDisplay; | 1085 bool _infraFailure; |
1059 | 1086 |
1060 BrowserCommandOutputImpl( | 1087 BrowserCommandOutputImpl( |
1061 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped) | 1088 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped) |
1062 : super(command, exitCode, timedOut, stdout, stderr, time, | 1089 : super(command, exitCode, timedOut, stdout, stderr, time, |
1063 compilationSkipped, 0) { | 1090 compilationSkipped, 0), |
1064 _failedBecauseOfMissingXDisplay = _didFailBecauseOfMissingXDisplay(); | 1091 _infraFailure = _failedBecauseOfFlakyInfrastructure(stderr); |
1065 if (_failedBecauseOfMissingXDisplay) { | |
1066 DebugLogger.warning("Warning: Test failure because of missing XDisplay"); | |
1067 // If we get the X server error, or DRT crashes with a core dump, retry | |
1068 // the test. | |
1069 } | |
1070 } | |
1071 | 1092 |
1072 Expectation result(TestCase testCase) { | 1093 Expectation result(TestCase testCase) { |
1073 // Handle crashes and timeouts first | 1094 // Handle crashes and timeouts first |
1074 if (hasCrashed) return Expectation.CRASH; | 1095 if (hasCrashed) return Expectation.CRASH; |
1075 if (hasTimedOut) return Expectation.TIMEOUT; | 1096 if (hasTimedOut) return Expectation.TIMEOUT; |
1076 | 1097 |
1098 if (_infraFailure) { | |
1099 return Expectation.IGNORE; | |
1100 } | |
1077 var outcome = _getOutcome(); | 1101 var outcome = _getOutcome(); |
1078 | 1102 |
1079 if (testCase.hasRuntimeError) { | 1103 if (testCase.hasRuntimeError) { |
1080 if (!outcome.canBeOutcomeOf(Expectation.RUNTIME_ERROR)) { | 1104 if (!outcome.canBeOutcomeOf(Expectation.RUNTIME_ERROR)) { |
1081 return Expectation.MISSING_RUNTIME_ERROR; | 1105 return Expectation.MISSING_RUNTIME_ERROR; |
1082 } | 1106 } |
1083 } | 1107 } |
1084 if (testCase.isNegative) { | 1108 if (testCase.isNegative) { |
1085 if (outcome.canBeOutcomeOf(Expectation.FAIL)) return Expectation.PASS; | 1109 if (outcome.canBeOutcomeOf(Expectation.FAIL)) return Expectation.PASS; |
1086 return Expectation.FAIL; | 1110 return Expectation.FAIL; |
1087 } | 1111 } |
1088 return outcome; | 1112 return outcome; |
1089 } | 1113 } |
1090 | 1114 |
1091 bool get successful => canRunDependendCommands; | 1115 bool get successful => canRunDependendCommands; |
1092 | 1116 |
1093 bool get canRunDependendCommands { | 1117 bool get canRunDependendCommands { |
1094 // We cannot rely on the exit code of content_shell as a method to determine | 1118 // We cannot rely on the exit code of content_shell as a method to |
1095 // if we were successful or not. | 1119 // determine if we were successful or not. |
1096 return super.canRunDependendCommands && !didFail(null); | 1120 return super.canRunDependendCommands && !didFail(null); |
1097 } | 1121 } |
1098 | 1122 |
1099 bool get hasCrashed { | 1123 bool get hasCrashed { |
1100 return super.hasCrashed || _rendererCrashed; | 1124 return super.hasCrashed || _rendererCrashed; |
1101 } | 1125 } |
1102 | 1126 |
1103 Expectation _getOutcome() { | 1127 Expectation _getOutcome() { |
1104 if (_failedBecauseOfMissingXDisplay) { | |
1105 return Expectation.FAIL; | |
1106 } | |
1107 | |
1108 if (_browserTestFailure) { | 1128 if (_browserTestFailure) { |
1109 return Expectation.RUNTIME_ERROR; | 1129 return Expectation.RUNTIME_ERROR; |
1110 } | 1130 } |
1111 return Expectation.PASS; | 1131 return Expectation.PASS; |
1112 } | 1132 } |
1113 | 1133 |
1114 bool _didFailBecauseOfMissingXDisplay() { | |
1115 // Browser case: | |
1116 // If the browser test failed, it may have been because content shell | |
1117 // and the virtual framebuffer X server didn't hook up, or it crashed with | |
1118 // a core dump. Sometimes content shell crashes after it has set the stdout | |
1119 // to PASS, so we have to do this check first. | |
1120 var stderrLines = decodeUtf8(super.stderr).split("\n"); | |
1121 for (String line in stderrLines) { | |
1122 // TODO(kustermann,ricow): Issue: 7564 | |
1123 // This seems to happen quite frequently, we need to figure out why. | |
1124 if (line.contains(MESSAGE_CANNOT_OPEN_DISPLAY) || | |
1125 line.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) { | |
1126 return true; | |
1127 } | |
1128 } | |
1129 return false; | |
1130 } | |
1131 | |
1132 bool get _rendererCrashed => | 1134 bool get _rendererCrashed => |
1133 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); | 1135 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); |
1134 | 1136 |
1135 bool get _browserTestFailure { | 1137 bool get _browserTestFailure { |
1136 // Browser tests fail unless stdout contains | 1138 // Browser tests fail unless stdout contains |
1137 // 'Content-Type: text/plain' followed by 'PASS'. | 1139 // 'Content-Type: text/plain' followed by 'PASS'. |
1138 bool hasContentType = false; | 1140 bool hasContentType = false; |
1139 var stdoutLines = decodeUtf8(super.stdout).split("\n"); | 1141 var stdoutLines = decodeUtf8(super.stdout).split("\n"); |
1140 var containsFail = false; | 1142 var containsFail = false; |
1141 var containsPass = false; | 1143 var containsPass = false; |
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1327 } | 1329 } |
1328 } | 1330 } |
1329 | 1331 |
1330 class BrowserControllerTestOutcome extends CommandOutputImpl | 1332 class BrowserControllerTestOutcome extends CommandOutputImpl |
1331 with UnittestSuiteMessagesMixin { | 1333 with UnittestSuiteMessagesMixin { |
1332 BrowserTestOutput _result; | 1334 BrowserTestOutput _result; |
1333 Expectation _rawOutcome; | 1335 Expectation _rawOutcome; |
1334 | 1336 |
1335 factory BrowserControllerTestOutcome( | 1337 factory BrowserControllerTestOutcome( |
1336 Command command, BrowserTestOutput result) { | 1338 Command command, BrowserTestOutput result) { |
1337 void validate(String assertion, bool value) { | |
1338 if (!value) { | |
1339 throw "InvalidFormat sent from browser driving page: $assertion:\n\n" | |
1340 "${result.lastKnownMessage}"; | |
1341 } | |
1342 } | |
1343 | |
1344 String indent(String string, int numSpaces) { | 1339 String indent(String string, int numSpaces) { |
1345 var spaces = new List.filled(numSpaces, ' ').join(''); | 1340 var spaces = new List.filled(numSpaces, ' ').join(''); |
1346 return string | 1341 return string |
1347 .replaceAll('\r\n', '\n') | 1342 .replaceAll('\r\n', '\n') |
1348 .split('\n') | 1343 .split('\n') |
1349 .map((line) => "$spaces$line") | 1344 .map((line) => "$spaces$line") |
1350 .join('\n'); | 1345 .join('\n'); |
1351 } | 1346 } |
1352 | 1347 |
1353 String stdout = ""; | 1348 String stdout = ""; |
(...skipping 13 matching lines...) Expand all Loading... | |
1367 } else { | 1362 } else { |
1368 outcome = Expectation.RUNTIME_ERROR; | 1363 outcome = Expectation.RUNTIME_ERROR; |
1369 } | 1364 } |
1370 } | 1365 } |
1371 | 1366 |
1372 if (result.didTimeout) { | 1367 if (result.didTimeout) { |
1373 if (result.delayUntilTestStarted != null) { | 1368 if (result.delayUntilTestStarted != null) { |
1374 stderr = "This test timed out. The delay until the test actually " | 1369 stderr = "This test timed out. The delay until the test actually " |
1375 "started was: ${result.delayUntilTestStarted}."; | 1370 "started was: ${result.delayUntilTestStarted}."; |
1376 } else { | 1371 } else { |
1377 // TODO(ricow/kustermann) as soon as we record the state periodically, | 1372 stderr = "This test has not notified test.py that it started running."; |
1378 // we will have more information and can remove this warning. | |
1379 stderr = "This test has not notified test.py that it started running. " | |
1380 "This could be a bug in test.py! " | |
1381 "Please contact ricow/whesse"; | |
1382 } | 1373 } |
1383 } | 1374 } |
1384 | 1375 |
1385 if (parsedResult != null) { | 1376 if (parsedResult != null) { |
1386 stdout = "events:\n${indent(prettifyJson(parsedResult.events), 2)}\n\n"; | 1377 stdout = "events:\n${indent(prettifyJson(parsedResult.events), 2)}\n\n"; |
1387 } else { | 1378 } else { |
1388 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; | 1379 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; |
1389 } | 1380 } |
1390 | 1381 |
1391 stderr = '$stderr\n\n' | 1382 stderr = '$stderr\n\n' |
(...skipping 1659 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
3051 } | 3042 } |
3052 } | 3043 } |
3053 | 3044 |
3054 void eventAllTestsDone() { | 3045 void eventAllTestsDone() { |
3055 for (var listener in _eventListener) { | 3046 for (var listener in _eventListener) { |
3056 listener.allDone(); | 3047 listener.allDone(); |
3057 } | 3048 } |
3058 _allDone(); | 3049 _allDone(); |
3059 } | 3050 } |
3060 } | 3051 } |
OLD | NEW |