Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(8)

Side by Side Diff: tools/testing/dart/test_runner.dart

Issue 2081173002: Add suppression to test.dart for flaky content_shell crashes (Closed) Base URL: git@github.com:dart-lang/sdk.git@master
Patch Set: Created 4 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /** 5 /**
6 * Classes and methods for executing tests. 6 * Classes and methods for executing tests.
7 * 7 *
8 * This module includes: 8 * This module includes:
9 * - Managing parallel execution of tests, including timeout checks. 9 * - Managing parallel execution of tests, including timeout checks.
10 * - Evaluating the output of each test as pass/fail/crash/timeout. 10 * - Evaluating the output of each test as pass/fail/crash/timeout.
(...skipping 1036 matching lines...) Expand 10 before | Expand all | Expand 10 after
1047 return Expectation.FAIL; 1047 return Expectation.FAIL;
1048 } 1048 }
1049 } 1049 }
1050 1050
1051 class BrowserCommandOutputImpl extends CommandOutputImpl { 1051 class BrowserCommandOutputImpl extends CommandOutputImpl {
1052 // Although tests are reported as passing, content shell sometimes exits with 1052 // Although tests are reported as passing, content shell sometimes exits with
1053 // a nonzero exitcode which makes our dartium builders extremely falky. 1053 // a nonzero exitcode which makes our dartium builders extremely falky.
1054 // See: http://dartbug.com/15139. 1054 // See: http://dartbug.com/15139.
1055 static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022; 1055 static int WHITELISTED_CONTENTSHELL_EXITCODE = -1073740022;
1056 static bool isWindows = io.Platform.operatingSystem == 'windows'; 1056 static bool isWindows = io.Platform.operatingSystem == 'windows';
1057 static bool _failedBecauseOfFlakyInfrastructure(List<int> stderr) {
1058 // If the browser test failed, it may have been because content shell
1059 // and the virtual framebuffer X server didn't hook up, or it crashed with
1060 // a core dump. Sometimes content shell crashes after it has set the stdout
1061 // to PASS, so we have to do this check first.
1062 // Content shell also fails with a broken pipe message: Issue 26739
1063 var zygoteCrash = new RegExp(
1064 r"ERROR:zygote_linux\.cc\(\d+\)] write: Broken pipe");
1065 var stderrLines = decodeUtf8(stderr).split("\n");
1066 for (String line in stderrLines) {
kustermann 2016/06/21 14:22:19 I think you can avoid the split into lines & itera
Bill Hesse 2016/06/24 12:06:59 Done.
1067 // TODO(whesse): Issue: 7564
1068 // This may not be happening anymore. Test by removing this suppression.
1069 if (line.contains(MESSAGE_CANNOT_OPEN_DISPLAY) ||
1070 line.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) {
1071 DebugLogger.warning(
1072 "Warning: Failure because of missing XDisplay. Test ignored");
1073 return true;
1074 }
1075 // Issue 26739
1076 if (zygoteCrash.hasMatch(line)) {
1077 DebugLogger.warning("Warning: Failure because of content_shell "
1078 "zygote crash. Test ignored");
1079 return true;
1080 }
1081 }
1082 return false;
1083 }
1057 1084
1058 bool _failedBecauseOfMissingXDisplay; 1085 bool _infraFailure;
1059 1086
1060 BrowserCommandOutputImpl( 1087 BrowserCommandOutputImpl(
1061 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped) 1088 command, exitCode, timedOut, stdout, stderr, time, compilationSkipped)
1062 : super(command, exitCode, timedOut, stdout, stderr, time, 1089 : super(command, exitCode, timedOut, stdout, stderr, time,
1063 compilationSkipped, 0) { 1090 compilationSkipped, 0),
1064 _failedBecauseOfMissingXDisplay = _didFailBecauseOfMissingXDisplay(); 1091 _infraFailure = _failedBecauseOfFlakyInfrastructure(stderr);
1065 if (_failedBecauseOfMissingXDisplay) {
1066 DebugLogger.warning("Warning: Test failure because of missing XDisplay");
1067 // If we get the X server error, or DRT crashes with a core dump, retry
1068 // the test.
1069 }
1070 }
1071 1092
1072 Expectation result(TestCase testCase) { 1093 Expectation result(TestCase testCase) {
1073 // Handle crashes and timeouts first 1094 // Handle crashes and timeouts first
1074 if (hasCrashed) return Expectation.CRASH; 1095 if (hasCrashed) return Expectation.CRASH;
1075 if (hasTimedOut) return Expectation.TIMEOUT; 1096 if (hasTimedOut) return Expectation.TIMEOUT;
1076 1097
1098 if (_infraFailure) {
1099 return Expectation.IGNORE;
1100 }
1077 var outcome = _getOutcome(); 1101 var outcome = _getOutcome();
1078 1102
1079 if (testCase.hasRuntimeError) { 1103 if (testCase.hasRuntimeError) {
1080 if (!outcome.canBeOutcomeOf(Expectation.RUNTIME_ERROR)) { 1104 if (!outcome.canBeOutcomeOf(Expectation.RUNTIME_ERROR)) {
1081 return Expectation.MISSING_RUNTIME_ERROR; 1105 return Expectation.MISSING_RUNTIME_ERROR;
1082 } 1106 }
1083 } 1107 }
1084 if (testCase.isNegative) { 1108 if (testCase.isNegative) {
1085 if (outcome.canBeOutcomeOf(Expectation.FAIL)) return Expectation.PASS; 1109 if (outcome.canBeOutcomeOf(Expectation.FAIL)) return Expectation.PASS;
1086 return Expectation.FAIL; 1110 return Expectation.FAIL;
1087 } 1111 }
1088 return outcome; 1112 return outcome;
1089 } 1113 }
1090 1114
1091 bool get successful => canRunDependendCommands; 1115 bool get successful => canRunDependendCommands;
1092 1116
1093 bool get canRunDependendCommands { 1117 bool get canRunDependendCommands {
1094 // We cannot rely on the exit code of content_shell as a method to determine 1118 // We cannot rely on the exit code of content_shell as a method to
1095 // if we were successful or not. 1119 // determine if we were successful or not.
1096 return super.canRunDependendCommands && !didFail(null); 1120 return super.canRunDependendCommands && !didFail(null);
1097 } 1121 }
1098 1122
1099 bool get hasCrashed { 1123 bool get hasCrashed {
1100 return super.hasCrashed || _rendererCrashed; 1124 return super.hasCrashed || _rendererCrashed;
1101 } 1125 }
1102 1126
1103 Expectation _getOutcome() { 1127 Expectation _getOutcome() {
1104 if (_failedBecauseOfMissingXDisplay) {
1105 return Expectation.FAIL;
1106 }
1107
1108 if (_browserTestFailure) { 1128 if (_browserTestFailure) {
1109 return Expectation.RUNTIME_ERROR; 1129 return Expectation.RUNTIME_ERROR;
1110 } 1130 }
1111 return Expectation.PASS; 1131 return Expectation.PASS;
1112 } 1132 }
1113 1133
1114 bool _didFailBecauseOfMissingXDisplay() {
1115 // Browser case:
1116 // If the browser test failed, it may have been because content shell
1117 // and the virtual framebuffer X server didn't hook up, or it crashed with
1118 // a core dump. Sometimes content shell crashes after it has set the stdout
1119 // to PASS, so we have to do this check first.
1120 var stderrLines = decodeUtf8(super.stderr).split("\n");
1121 for (String line in stderrLines) {
1122 // TODO(kustermann,ricow): Issue: 7564
1123 // This seems to happen quite frequently, we need to figure out why.
1124 if (line.contains(MESSAGE_CANNOT_OPEN_DISPLAY) ||
1125 line.contains(MESSAGE_FAILED_TO_RUN_COMMAND)) {
1126 return true;
1127 }
1128 }
1129 return false;
1130 }
1131
1132 bool get _rendererCrashed => 1134 bool get _rendererCrashed =>
1133 decodeUtf8(super.stdout).contains("#CRASHED - rendere"); 1135 decodeUtf8(super.stdout).contains("#CRASHED - rendere");
1134 1136
1135 bool get _browserTestFailure { 1137 bool get _browserTestFailure {
1136 // Browser tests fail unless stdout contains 1138 // Browser tests fail unless stdout contains
1137 // 'Content-Type: text/plain' followed by 'PASS'. 1139 // 'Content-Type: text/plain' followed by 'PASS'.
1138 bool hasContentType = false; 1140 bool hasContentType = false;
1139 var stdoutLines = decodeUtf8(super.stdout).split("\n"); 1141 var stdoutLines = decodeUtf8(super.stdout).split("\n");
1140 var containsFail = false; 1142 var containsFail = false;
1141 var containsPass = false; 1143 var containsPass = false;
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after
1327 } 1329 }
1328 } 1330 }
1329 1331
1330 class BrowserControllerTestOutcome extends CommandOutputImpl 1332 class BrowserControllerTestOutcome extends CommandOutputImpl
1331 with UnittestSuiteMessagesMixin { 1333 with UnittestSuiteMessagesMixin {
1332 BrowserTestOutput _result; 1334 BrowserTestOutput _result;
1333 Expectation _rawOutcome; 1335 Expectation _rawOutcome;
1334 1336
1335 factory BrowserControllerTestOutcome( 1337 factory BrowserControllerTestOutcome(
1336 Command command, BrowserTestOutput result) { 1338 Command command, BrowserTestOutput result) {
1337 void validate(String assertion, bool value) {
1338 if (!value) {
1339 throw "InvalidFormat sent from browser driving page: $assertion:\n\n"
1340 "${result.lastKnownMessage}";
1341 }
1342 }
1343
1344 String indent(String string, int numSpaces) { 1339 String indent(String string, int numSpaces) {
1345 var spaces = new List.filled(numSpaces, ' ').join(''); 1340 var spaces = new List.filled(numSpaces, ' ').join('');
1346 return string 1341 return string
1347 .replaceAll('\r\n', '\n') 1342 .replaceAll('\r\n', '\n')
1348 .split('\n') 1343 .split('\n')
1349 .map((line) => "$spaces$line") 1344 .map((line) => "$spaces$line")
1350 .join('\n'); 1345 .join('\n');
1351 } 1346 }
1352 1347
1353 String stdout = ""; 1348 String stdout = "";
(...skipping 13 matching lines...) Expand all
1367 } else { 1362 } else {
1368 outcome = Expectation.RUNTIME_ERROR; 1363 outcome = Expectation.RUNTIME_ERROR;
1369 } 1364 }
1370 } 1365 }
1371 1366
1372 if (result.didTimeout) { 1367 if (result.didTimeout) {
1373 if (result.delayUntilTestStarted != null) { 1368 if (result.delayUntilTestStarted != null) {
1374 stderr = "This test timed out. The delay until the test actually " 1369 stderr = "This test timed out. The delay until the test actually "
1375 "started was: ${result.delayUntilTestStarted}."; 1370 "started was: ${result.delayUntilTestStarted}.";
1376 } else { 1371 } else {
1377 // TODO(ricow/kustermann) as soon as we record the state periodically, 1372 stderr = "This test has not notified test.py that it started running.";
1378 // we will have more information and can remove this warning.
1379 stderr = "This test has not notified test.py that it started running. "
1380 "This could be a bug in test.py! "
1381 "Please contact ricow/whesse";
1382 } 1373 }
1383 } 1374 }
1384 1375
1385 if (parsedResult != null) { 1376 if (parsedResult != null) {
1386 stdout = "events:\n${indent(prettifyJson(parsedResult.events), 2)}\n\n"; 1377 stdout = "events:\n${indent(prettifyJson(parsedResult.events), 2)}\n\n";
1387 } else { 1378 } else {
1388 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n"; 1379 stdout = "message:\n${indent(result.lastKnownMessage, 2)}\n\n";
1389 } 1380 }
1390 1381
1391 stderr = '$stderr\n\n' 1382 stderr = '$stderr\n\n'
(...skipping 1659 matching lines...) Expand 10 before | Expand all | Expand 10 after
3051 } 3042 }
3052 } 3043 }
3053 3044
3054 void eventAllTestsDone() { 3045 void eventAllTestsDone() {
3055 for (var listener in _eventListener) { 3046 for (var listener in _eventListener) {
3056 listener.allDone(); 3047 listener.allDone();
3057 } 3048 }
3058 _allDone(); 3049 _allDone();
3059 } 3050 }
3060 } 3051 }
OLDNEW
« tools/testing/dart/status_file_parser.dart ('K') | « tools/testing/dart/test_configurations.dart ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698