OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for executing tests. | 6 * Classes and methods for executing tests. |
7 * | 7 * |
8 * This module includes: | 8 * This module includes: |
9 * - Managing parallel execution of tests, including timeout checks. | 9 * - Managing parallel execution of tests, including timeout checks. |
10 * - Evaluating the output of each test as pass/fail/crash/timeout. | 10 * - Evaluating the output of each test as pass/fail/crash/timeout. |
11 */ | 11 */ |
12 library test_runner; | 12 library test_runner; |
13 | 13 |
14 import "dart:async"; | 14 import "dart:async"; |
15 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow | 15 // We need to use the 'io' prefix here, otherwise io.exitCode will shadow |
16 // CommandOutput.exitCode in subclasses of CommandOutput. | 16 // CommandOutput.exitCode in subclasses of CommandOutput. |
17 import "dart:io" as io; | 17 import "dart:io" as io; |
18 import "dart:isolate"; | 18 import "dart:isolate"; |
19 import "dart:uri"; | 19 import "dart:uri"; |
| 20 import "http_server.dart" as http_server; |
20 import "status_file_parser.dart"; | 21 import "status_file_parser.dart"; |
21 import "test_progress.dart"; | 22 import "test_progress.dart"; |
22 import "test_suite.dart"; | 23 import "test_suite.dart"; |
23 import "http_server.dart" as http_server; | 24 import "utils.dart"; |
24 | 25 |
25 const int NO_TIMEOUT = 0; | 26 const int NO_TIMEOUT = 0; |
26 const int SLOW_TIMEOUT_MULTIPLIER = 4; | 27 const int SLOW_TIMEOUT_MULTIPLIER = 4; |
27 | 28 |
28 typedef void TestCaseEvent(TestCase testCase); | 29 typedef void TestCaseEvent(TestCase testCase); |
29 typedef void ExitCodeEvent(int exitCode); | 30 typedef void ExitCodeEvent(int exitCode); |
30 typedef void EnqueueMoreWork(ProcessQueue queue); | 31 typedef void EnqueueMoreWork(ProcessQueue queue); |
31 | 32 |
32 | 33 |
33 /** | 34 /** |
(...skipping 584 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
618 } | 619 } |
619 return _browserTestFailure; | 620 return _browserTestFailure; |
620 } | 621 } |
621 | 622 |
622 bool get _failedBecauseOfMissingXDisplay { | 623 bool get _failedBecauseOfMissingXDisplay { |
623 // Browser case: | 624 // Browser case: |
624 // If the browser test failed, it may have been because DumpRenderTree | 625 // If the browser test failed, it may have been because DumpRenderTree |
625 // and the virtual framebuffer X server didn't hook up, or DRT crashed with | 626 // and the virtual framebuffer X server didn't hook up, or DRT crashed with |
626 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, | 627 // a core dump. Sometimes DRT crashes after it has set the stdout to PASS, |
627 // so we have to do this check first. | 628 // so we have to do this check first. |
628 var stderrLines = new String.fromCharCodes(super.stderr).split("\n"); | 629 var stderrLines = decodeUtf8(super.stderr).split("\n"); |
629 for (String line in stderrLines) { | 630 for (String line in stderrLines) { |
630 // TODO(kustermann,ricow): Issue: 7564 | 631 // TODO(kustermann,ricow): Issue: 7564 |
631 // This seems to happen quite frequently, we need to figure out why. | 632 // This seems to happen quite frequently, we need to figure out why. |
632 if (line.contains('Gtk-WARNING **: cannot open display') || | 633 if (line.contains('Gtk-WARNING **: cannot open display') || |
633 line.contains('Failed to run command. return code=1')) { | 634 line.contains('Failed to run command. return code=1')) { |
634 // If we get the X server error, or DRT crashes with a core dump, retry | 635 // If we get the X server error, or DRT crashes with a core dump, retry |
635 // the test. | 636 // the test. |
636 if ((testCase as BrowserTestCase).numRetries > 0) { | 637 if ((testCase as BrowserTestCase).numRetries > 0) { |
637 requestRetry = true; | 638 requestRetry = true; |
638 } | 639 } |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
693 stdout.length); | 694 stdout.length); |
694 } | 695 } |
695 } | 696 } |
696 return true; | 697 return true; |
697 } | 698 } |
698 | 699 |
699 bool get _browserTestFailure { | 700 bool get _browserTestFailure { |
700 // Browser tests fail unless stdout contains | 701 // Browser tests fail unless stdout contains |
701 // 'Content-Type: text/plain' followed by 'PASS'. | 702 // 'Content-Type: text/plain' followed by 'PASS'. |
702 bool has_content_type = false; | 703 bool has_content_type = false; |
703 var stdoutLines = new String.fromCharCodes(super.stdout).split("\n"); | 704 var stdoutLines = decodeUtf8(super.stdout).split("\n"); |
704 for (String line in stdoutLines) { | 705 for (String line in stdoutLines) { |
705 switch (line) { | 706 switch (line) { |
706 case 'Content-Type: text/plain': | 707 case 'Content-Type: text/plain': |
707 has_content_type = true; | 708 has_content_type = true; |
708 break; | 709 break; |
709 case 'PASS': | 710 case 'PASS': |
710 if (has_content_type) { | 711 if (has_content_type) { |
711 if (exitCode != 0) { | 712 if (exitCode != 0) { |
712 print("Warning: All tests passed, but exitCode != 0 " | 713 print("Warning: All tests passed, but exitCode != 0 " |
713 "(${testCase.displayName})"); | 714 "(${testCase.displayName})"); |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
766 return failResult; | 767 return failResult; |
767 } | 768 } |
768 | 769 |
769 bool _didFail() { | 770 bool _didFail() { |
770 if (hasCrashed) return false; | 771 if (hasCrashed) return false; |
771 | 772 |
772 List<String> errors = []; | 773 List<String> errors = []; |
773 List<String> staticWarnings = []; | 774 List<String> staticWarnings = []; |
774 | 775 |
775 // Read the returned list of errors and stuff them away. | 776 // Read the returned list of errors and stuff them away. |
776 var stderrLines = new String.fromCharCodes(super.stderr).split("\n"); | 777 var stderrLines = decodeUtf8(super.stderr).split("\n"); |
777 for (String line in stderrLines) { | 778 for (String line in stderrLines) { |
778 if (line.length == 0) continue; | 779 if (line.length == 0) continue; |
779 List<String> fields = splitMachineError(line); | 780 List<String> fields = splitMachineError(line); |
780 if (fields[ERROR_LEVEL] == 'ERROR') { | 781 if (fields[ERROR_LEVEL] == 'ERROR') { |
781 errors.add(fields[FORMATTED_ERROR]); | 782 errors.add(fields[FORMATTED_ERROR]); |
782 } else if (fields[ERROR_LEVEL] == 'WARNING') { | 783 } else if (fields[ERROR_LEVEL] == 'WARNING') { |
783 // We only care about testing Static type warnings | 784 // We only care about testing Static type warnings |
784 // ignore all others | 785 // ignore all others |
785 if (fields[ERROR_TYPE] == 'STATIC_TYPE') { | 786 if (fields[ERROR_TYPE] == 'STATIC_TYPE') { |
786 staticWarnings.add(fields[FORMATTED_ERROR]); | 787 staticWarnings.add(fields[FORMATTED_ERROR]); |
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
942 var command = lastCommandOutput.command; | 943 var command = lastCommandOutput.command; |
943 | 944 |
944 if (timeoutTimer != null) { | 945 if (timeoutTimer != null) { |
945 timeoutTimer.cancel(); | 946 timeoutTimer.cancel(); |
946 } | 947 } |
947 if (lastCommandOutput.unexpectedOutput | 948 if (lastCommandOutput.unexpectedOutput |
948 && testCase.configuration['verbose'] != null | 949 && testCase.configuration['verbose'] != null |
949 && testCase.configuration['verbose']) { | 950 && testCase.configuration['verbose']) { |
950 print(testCase.displayName); | 951 print(testCase.displayName); |
951 | 952 |
952 print(new String.fromCharCodes(lastCommandOutput.stderr)); | 953 print(decodeUtf8(lastCommandOutput.stderr)); |
953 if (!lastCommandOutput.command.isPixelTest) { | 954 if (!lastCommandOutput.command.isPixelTest) { |
954 print(new String.fromCharCodes(lastCommandOutput.stdout)); | 955 print(decodeUtf8(lastCommandOutput.stdout)); |
955 } else { | 956 } else { |
956 print("DRT pixel test failed! stdout is not printed because it " | 957 print("DRT pixel test failed! stdout is not printed because it " |
957 "contains binary data!"); | 958 "contains binary data!"); |
958 } | 959 } |
959 print(''); | 960 print(''); |
960 if (notifications.length > 0) { | 961 if (notifications.length > 0) { |
961 print("Notifications:"); | 962 print("Notifications:"); |
962 for (var line in notifications) { | 963 for (var line in notifications) { |
963 print(notifications); | 964 print(notifications); |
964 } | 965 } |
(...skipping 347 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1312 // Otherwise, process output and call _reportResult() when done. | 1313 // Otherwise, process output and call _reportResult() when done. |
1313 var line = stream.readLine(); | 1314 var line = stream.readLine(); |
1314 while (line != null) { | 1315 while (line != null) { |
1315 if (line.startsWith('>>> TEST')) { | 1316 if (line.startsWith('>>> TEST')) { |
1316 _status = line; | 1317 _status = line; |
1317 } else if (line.startsWith('>>> BATCH START')) { | 1318 } else if (line.startsWith('>>> BATCH START')) { |
1318 // ignore | 1319 // ignore |
1319 } else if (line.startsWith('>>> ')) { | 1320 } else if (line.startsWith('>>> ')) { |
1320 throw new Exception('Unexpected command from dartc batch runner.'); | 1321 throw new Exception('Unexpected command from dartc batch runner.'); |
1321 } else { | 1322 } else { |
1322 buffer.addAll("$line\n".charCodes); | 1323 buffer.addAll(decodeUtf8(line)); |
| 1324 buffer.addAll("\n".charCodes); |
1323 } | 1325 } |
1324 line = stream.readLine(); | 1326 line = stream.readLine(); |
1325 } | 1327 } |
1326 if (_status != null) { | 1328 if (_status != null) { |
1327 _timer.cancel(); | 1329 _timer.cancel(); |
1328 _stdoutDone(); | 1330 _stdoutDone(); |
1329 } | 1331 } |
1330 } | 1332 } |
1331 stream.onLine = onLineHandler; | 1333 stream.onLine = onLineHandler; |
1332 } | 1334 } |
1333 | 1335 |
1334 void _readStderr(io.StringInputStream stream, List<int> buffer) { | 1336 void _readStderr(io.StringInputStream stream, List<int> buffer) { |
1335 var ignoreStreams = _ignoreStreams; // Capture this mutable object. | 1337 var ignoreStreams = _ignoreStreams; // Capture this mutable object. |
1336 void onLineHandler() { | 1338 void onLineHandler() { |
1337 if (ignoreStreams.value) { | 1339 if (ignoreStreams.value) { |
1338 while (stream.readLine() != null) { | 1340 while (stream.readLine() != null) { |
1339 // Do nothing. | 1341 // Do nothing. |
1340 } | 1342 } |
1341 return; | 1343 return; |
1342 } | 1344 } |
1343 // Otherwise, process output and call _reportResult() when done. | 1345 // Otherwise, process output and call _reportResult() when done. |
1344 var line = stream.readLine(); | 1346 var line = stream.readLine(); |
1345 while (line != null) { | 1347 while (line != null) { |
1346 if (line.startsWith('>>> EOF STDERR')) { | 1348 if (line.startsWith('>>> EOF STDERR')) { |
1347 _stderrDone(); | 1349 _stderrDone(); |
1348 } else { | 1350 } else { |
1349 buffer.addAll("$line\n".charCodes); | 1351 buffer.addAll(decodeUtf8(line)); |
| 1352 buffer.addAll("\n".charCodes); |
1350 } | 1353 } |
1351 line = stream.readLine(); | 1354 line = stream.readLine(); |
1352 } | 1355 } |
1353 } | 1356 } |
1354 stream.onLine = onLineHandler; | 1357 stream.onLine = onLineHandler; |
1355 } | 1358 } |
1356 | 1359 |
1357 ExitCodeEvent makeExitHandler(String status) { | 1360 ExitCodeEvent makeExitHandler(String status) { |
1358 void handler(int exitCode) { | 1361 void handler(int exitCode) { |
1359 if (active) { | 1362 if (active) { |
(...skipping 388 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1748 // the developer doesn't waste his or her time trying to fix a bunch of | 1751 // the developer doesn't waste his or her time trying to fix a bunch of |
1749 // tests that appear to be broken but were actually just flakes that | 1752 // tests that appear to be broken but were actually just flakes that |
1750 // didn't get retried because there had already been one failure. | 1753 // didn't get retried because there had already been one failure. |
1751 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; | 1754 bool allowRetry = _MAX_FAILED_NO_RETRY > _progress.numFailedTests; |
1752 new RunningProcess(test, allowRetry, this).start(); | 1755 new RunningProcess(test, allowRetry, this).start(); |
1753 } | 1756 } |
1754 _numProcesses++; | 1757 _numProcesses++; |
1755 } | 1758 } |
1756 } | 1759 } |
1757 } | 1760 } |
OLD | NEW |