Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library test_progress; | 5 library test_progress; |
| 6 | 6 |
| 7 import "dart:convert" show JSON; | |
| 7 import "dart:io"; | 8 import "dart:io"; |
| 8 import "dart:io" as io; | 9 |
| 9 import "dart:convert" show JSON; | |
| 10 import "path.dart"; | 10 import "path.dart"; |
| 11 import "status_file_parser.dart"; | 11 import "status_file_parser.dart"; |
| 12 import "summary_report.dart"; | 12 import "summary_report.dart"; |
| 13 import "test_runner.dart"; | 13 import "test_runner.dart"; |
| 14 import "test_suite.dart"; | 14 import "test_suite.dart"; |
| 15 import "utils.dart"; | 15 import "utils.dart"; |
| 16 | 16 |
| 17 String _pad(String s, int length) { | 17 /// Controls how message strings are processed before being displayed. |
| 18 StringBuffer buffer = new StringBuffer(); | 18 class Formatter { |
| 19 for (int i = s.length; i < length; i++) { | 19 /// Messages are left as-is. |
| 20 buffer.write(' '); | 20 static const normal = const Formatter._(); |
| 21 } | 21 |
| 22 buffer.write(s); | 22 /// Messages are wrapped in ANSI escape codes to color them for display on a |
| 23 return buffer.toString(); | 23 /// terminal. |
| 24 static const color = const _ColorFormatter(); | |
| 25 | |
| 26 const Formatter._(); | |
| 27 | |
| 28 /// Formats a success message. | |
| 29 String passed(String message) => message; | |
| 30 | |
| 31 /// Formats a failure message. | |
| 32 String failed(String message) => message; | |
| 24 } | 33 } |
| 25 | 34 |
| 26 String _padTime(int time) { | 35 class _ColorFormatter extends Formatter { |
| 27 if (time == 0) { | 36 static const _green = 32; |
| 28 return '00'; | 37 static const _red = 31; |
| 29 } else if (time < 10) { | 38 static const _escape = '\u001b'; |
| 30 return '0$time'; | |
| 31 } else { | |
| 32 return '$time'; | |
| 33 } | |
| 34 } | |
| 35 | 39 |
| 36 String _timeString(Duration d) { | 40 const _ColorFormatter() : super._(); |
| 37 var min = d.inMinutes; | |
| 38 var sec = d.inSeconds % 60; | |
| 39 return '${_padTime(min)}:${_padTime(sec)}'; | |
| 40 } | |
| 41 | 41 |
| 42 class Formatter { | 42 String passed(String message) => _color(message, _green); |
| 43 const Formatter(); | 43 String failed(String message) => _color(message, _red); |
| 44 String passed(msg) => msg; | |
| 45 String failed(msg) => msg; | |
| 46 } | |
| 47 | 44 |
| 48 class ColorFormatter extends Formatter { | 45 static String _color(String message, int color) => |
| 49 static int BOLD = 1; | 46 "$_escape[${color}m$message$_escape[0m"; |
| 50 static int GREEN = 32; | |
| 51 static int RED = 31; | |
| 52 static int NONE = 0; | |
| 53 static String ESCAPE = decodeUtf8([27]); | |
| 54 | |
| 55 String passed(String msg) => _color(msg, GREEN); | |
| 56 String failed(String msg) => _color(msg, RED); | |
| 57 | |
| 58 static String _color(String msg, int color) { | |
| 59 return "$ESCAPE[${color}m$msg$ESCAPE[0m"; | |
| 60 } | |
| 61 } | |
| 62 | |
| 63 List<String> _buildFailureOutput(TestCase test, | |
| 64 [Formatter formatter = const Formatter()]) { | |
| 65 List<String> getLinesWithoutCarriageReturn(List<int> output) { | |
| 66 return decodeUtf8(output) | |
| 67 .replaceAll('\r\n', '\n') | |
| 68 .replaceAll('\r', '\n') | |
| 69 .split('\n'); | |
| 70 } | |
| 71 | |
| 72 List<String> output = new List<String>(); | |
| 73 output.add(''); | |
| 74 output.add(formatter.failed('FAILED: ${test.configurationString}' | |
| 75 ' ${test.displayName}')); | |
| 76 StringBuffer expected = new StringBuffer(); | |
| 77 expected.write('Expected: '); | |
| 78 for (var expectation in test.expectedOutcomes) { | |
| 79 expected.write('$expectation '); | |
| 80 } | |
| 81 output.add(expected.toString()); | |
| 82 output.add('Actual: ${test.result}'); | |
| 83 if (!test.lastCommandOutput.hasTimedOut) { | |
| 84 if (test.commandOutputs.length != test.commands.length && | |
| 85 !test.expectCompileError) { | |
| 86 output.add('Unexpected compile-time error.'); | |
| 87 } else { | |
| 88 if (test.expectCompileError) { | |
| 89 output.add('Compile-time error expected.'); | |
| 90 } | |
| 91 if (test.hasRuntimeError) { | |
| 92 output.add('Runtime error expected.'); | |
| 93 } | |
| 94 if (test.configuration['checked'] && test.isNegativeIfChecked) { | |
| 95 output.add('Dynamic type error expected.'); | |
| 96 } | |
| 97 } | |
| 98 } | |
| 99 for (var i = 0; i < test.commands.length; i++) { | |
| 100 var command = test.commands[i]; | |
| 101 var commandOutput = test.commandOutputs[command]; | |
| 102 if (commandOutput != null) { | |
| 103 output.add("CommandOutput[${command.displayName}]:"); | |
| 104 if (!commandOutput.diagnostics.isEmpty) { | |
| 105 String prefix = 'diagnostics:'; | |
| 106 for (var s in commandOutput.diagnostics) { | |
| 107 output.add('$prefix ${s}'); | |
| 108 prefix = ' '; | |
| 109 } | |
| 110 } | |
| 111 if (!commandOutput.stdout.isEmpty) { | |
| 112 output.add(''); | |
| 113 output.add('stdout:'); | |
| 114 output.addAll(getLinesWithoutCarriageReturn(commandOutput.stdout)); | |
| 115 } | |
| 116 if (!commandOutput.stderr.isEmpty) { | |
| 117 output.add(''); | |
| 118 output.add('stderr:'); | |
| 119 output.addAll(getLinesWithoutCarriageReturn(commandOutput.stderr)); | |
| 120 } | |
| 121 } | |
| 122 } | |
| 123 if (test is BrowserTestCase) { | |
| 124 // Additional command for rerunning the steps locally after the fact. | |
| 125 var command = test.configuration["_servers_"].httpServerCommandline(); | |
| 126 output.add(''); | |
| 127 output.add('To retest, run: $command'); | |
| 128 } | |
| 129 for (var i = 0; i < test.commands.length; i++) { | |
| 130 var command = test.commands[i]; | |
| 131 var commandOutput = test.commandOutputs[command]; | |
| 132 output.add(''); | |
| 133 output.add('Command[${command.displayName}]: $command'); | |
| 134 if (commandOutput != null) { | |
| 135 output.add('Took ${commandOutput.time}'); | |
| 136 } else { | |
| 137 output.add('Did not run'); | |
| 138 } | |
| 139 } | |
| 140 | |
| 141 var arguments = ['python', 'tools/test.py']; | |
| 142 arguments.addAll(test.configuration['_reproducing_arguments_']); | |
| 143 arguments.add(test.displayName); | |
| 144 var testPyCommandline = arguments.map(escapeCommandLineArgument).join(' '); | |
| 145 | |
| 146 output.add(''); | |
| 147 output.add('Short reproduction command (experimental):'); | |
| 148 output.add(" $testPyCommandline"); | |
| 149 return output; | |
| 150 } | |
| 151 | |
| 152 String _buildSummaryEnd(int failedTests) { | |
| 153 if (failedTests == 0) { | |
| 154 return '\n===\n=== All tests succeeded\n===\n'; | |
| 155 } else { | |
| 156 var pluralSuffix = failedTests != 1 ? 's' : ''; | |
| 157 return '\n===\n=== ${failedTests} test$pluralSuffix failed\n===\n'; | |
| 158 } | |
| 159 } | 47 } |
| 160 | 48 |
| 161 class EventListener { | 49 class EventListener { |
| 162 void testAdded() {} | 50 void testAdded() {} |
| 163 void done(TestCase test) {} | 51 void done(TestCase test) {} |
| 164 void allTestsKnown() {} | 52 void allTestsKnown() {} |
| 165 void allDone() {} | 53 void allDone() {} |
| 166 } | 54 } |
| 167 | 55 |
| 168 class ExitCodeSetter extends EventListener { | 56 class ExitCodeSetter extends EventListener { |
| 169 void done(TestCase test) { | 57 void done(TestCase test) { |
| 170 if (test.unexpectedOutput) { | 58 if (test.unexpectedOutput) { |
| 171 io.exitCode = 1; | 59 exitCode = 1; |
| 172 } | 60 } |
| 173 } | 61 } |
| 174 } | 62 } |
| 175 | 63 |
| 176 class IgnoredTestMonitor extends EventListener { | 64 class IgnoredTestMonitor extends EventListener { |
| 177 static final int maxIgnored = 5; | 65 static final int maxIgnored = 5; |
| 178 | 66 |
| 179 int countIgnored = 0; | 67 int countIgnored = 0; |
| 180 | 68 |
| 181 void done(TestCase test) { | 69 void done(TestCase test) { |
| (...skipping 21 matching lines...) Expand all Loading... | |
| 203 if (test.isFlaky && test.result != Expectation.PASS) { | 91 if (test.isFlaky && test.result != Expectation.PASS) { |
| 204 var buf = new StringBuffer(); | 92 var buf = new StringBuffer(); |
| 205 for (var l in _buildFailureOutput(test)) { | 93 for (var l in _buildFailureOutput(test)) { |
| 206 buf.write("$l\n"); | 94 buf.write("$l\n"); |
| 207 } | 95 } |
| 208 _appendToFlakyFile(buf.toString()); | 96 _appendToFlakyFile(buf.toString()); |
| 209 } | 97 } |
| 210 } | 98 } |
| 211 | 99 |
| 212 void _appendToFlakyFile(String msg) { | 100 void _appendToFlakyFile(String msg) { |
| 213 var file = new File(TestUtils.flakyFileName()); | 101 var file = new File(TestUtils.flakyFileName); |
| 214 var fd = file.openSync(mode: FileMode.APPEND); | 102 var fd = file.openSync(mode: FileMode.APPEND); |
| 215 fd.writeStringSync(msg); | 103 fd.writeStringSync(msg); |
| 216 fd.closeSync(); | 104 fd.closeSync(); |
| 217 } | 105 } |
| 218 } | 106 } |
| 219 | 107 |
| 220 class TestOutcomeLogWriter extends EventListener { | 108 class TestOutcomeLogWriter extends EventListener { |
| 221 /* | 109 /* |
| 222 * The ".test-outcome.log" file contains one line for every executed test. | 110 * The ".test-outcome.log" file contains one line for every executed test. |
| 223 * Such a line is an encoded JSON data structure of the following form: | 111 * Such a line is an encoded JSON data structure of the following form: |
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 300 }, | 188 }, |
| 301 }); | 189 }); |
| 302 } | 190 } |
| 303 | 191 |
| 304 void allDone() { | 192 void allDone() { |
| 305 if (_sink != null) _sink.close(); | 193 if (_sink != null) _sink.close(); |
| 306 } | 194 } |
| 307 | 195 |
| 308 void _writeTestOutcomeRecord(Map record) { | 196 void _writeTestOutcomeRecord(Map record) { |
| 309 if (_sink == null) { | 197 if (_sink == null) { |
| 310 _sink = new File(TestUtils.testOutcomeFileName()) | 198 _sink = new File(TestUtils.testOutcomeFileName) |
| 311 .openWrite(mode: FileMode.APPEND); | 199 .openWrite(mode: FileMode.APPEND); |
| 312 } | 200 } |
| 313 _sink.write("${JSON.encode(record)}\n"); | 201 _sink.write("${JSON.encode(record)}\n"); |
| 314 } | 202 } |
| 315 } | 203 } |
| 316 | 204 |
| 317 class UnexpectedCrashLogger extends EventListener { | 205 class UnexpectedCrashLogger extends EventListener { |
| 318 final archivedBinaries = <String, String>{}; | 206 final archivedBinaries = <String, String>{}; |
| 319 | 207 |
| 320 void done(TestCase test) { | 208 void done(TestCase test) { |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 489 | 377 |
| 490 void done(TestCase test) { | 378 void done(TestCase test) { |
| 491 for (var commandOutput in test.commandOutputs.values) { | 379 for (var commandOutput in test.commandOutputs.values) { |
| 492 if (commandOutput.compilationSkipped) _skippedCompilations++; | 380 if (commandOutput.compilationSkipped) _skippedCompilations++; |
| 493 } | 381 } |
| 494 } | 382 } |
| 495 | 383 |
| 496 void allDone() { | 384 void allDone() { |
| 497 if (_skippedCompilations > 0) { | 385 if (_skippedCompilations > 0) { |
| 498 print('\n$_skippedCompilations compilations were skipped because ' | 386 print('\n$_skippedCompilations compilations were skipped because ' |
| 499 'the previous output was already up to date\n'); | 387 'the previous output was already up to date.\n'); |
| 500 } | 388 } |
| 501 } | 389 } |
| 502 } | 390 } |
| 503 | 391 |
| 504 class LineProgressIndicator extends EventListener { | 392 class LineProgressIndicator extends EventListener { |
| 505 void done(TestCase test) { | 393 void done(TestCase test) { |
| 506 var status = 'pass'; | 394 var status = 'pass'; |
| 507 if (test.unexpectedOutput) { | 395 if (test.unexpectedOutput) { |
| 508 status = 'fail'; | 396 status = 'fail'; |
| 509 } | 397 } |
| 510 print('Done ${test.configurationString} ${test.displayName}: $status'); | 398 print('Done ${test.configurationString} ${test.displayName}: $status'); |
| 511 } | 399 } |
| 512 } | 400 } |
| 513 | 401 |
| 514 class TestFailurePrinter extends EventListener { | 402 class TestFailurePrinter extends EventListener { |
| 515 bool _printSummary; | 403 final bool _printSummary; |
| 516 var _formatter; | 404 final Formatter _formatter; |
| 517 var _failureSummary = <String>[]; | 405 final _failureSummary = <String>[]; |
| 518 var _failedTests = 0; | 406 int _failedTests = 0; |
| 519 | 407 |
| 520 TestFailurePrinter(this._printSummary, [this._formatter = const Formatter()]); | 408 TestFailurePrinter(this._printSummary, [this._formatter = Formatter.normal]); |
| 521 | 409 |
| 522 void done(TestCase test) { | 410 void done(TestCase test) { |
| 523 if (test.unexpectedOutput) { | 411 if (test.unexpectedOutput) { |
| 524 _failedTests++; | 412 _failedTests++; |
| 525 var lines = _buildFailureOutput(test, _formatter); | 413 var lines = _buildFailureOutput(test, _formatter); |
| 526 for (var line in lines) { | 414 for (var line in lines) { |
| 527 print(line); | 415 print(line); |
| 528 } | 416 } |
| 529 print(''); | 417 print(''); |
| 530 if (_printSummary) { | 418 if (_printSummary) { |
| 531 _failureSummary.addAll(lines); | 419 _failureSummary.addAll(lines); |
| 532 _failureSummary.add(''); | 420 _failureSummary.add(''); |
| 533 } | 421 } |
| 534 } | 422 } |
| 535 } | 423 } |
| 536 | 424 |
| 537 void allDone() { | 425 void allDone() { |
| 538 if (_printSummary) { | 426 if (_printSummary) { |
| 539 if (!_failureSummary.isEmpty) { | 427 if (!_failureSummary.isEmpty) { |
| 540 print('\n=== Failure summary:\n'); | 428 print('\n=== Failure summary:\n'); |
| 541 for (String line in _failureSummary) { | 429 for (var line in _failureSummary) { |
| 542 print(line); | 430 print(line); |
| 543 } | 431 } |
| 544 print(''); | 432 print(''); |
| 545 | 433 |
| 546 print(_buildSummaryEnd(_failedTests)); | 434 print(_buildSummaryEnd(_failedTests)); |
| 547 } | 435 } |
| 548 } | 436 } |
| 549 } | 437 } |
| 550 } | 438 } |
| 551 | 439 |
| 552 class ProgressIndicator extends EventListener { | 440 class ProgressIndicator extends EventListener { |
| 553 ProgressIndicator(this._startTime); | 441 ProgressIndicator(this._startTime); |
| 554 | 442 |
| 443 static EventListener fromName( | |
| 444 String name, DateTime startTime, Formatter formatter) { | |
| 445 switch (name) { | |
| 446 case 'compact': | |
| 447 return new CompactProgressIndicator(startTime, formatter); | |
| 448 case 'line': | |
| 449 return new LineProgressIndicator(); | |
| 450 case 'verbose': | |
| 451 return new VerboseProgressIndicator(startTime); | |
| 452 case 'status': | |
| 453 return new ProgressIndicator(startTime); | |
| 454 case 'buildbot': | |
| 455 return new BuildbotProgressIndicator(startTime); | |
| 456 default: | |
| 457 throw new ArgumentError('Unknown progress indicator "$name".'); | |
| 458 } | |
| 459 } | |
| 460 | |
| 555 void testAdded() { | 461 void testAdded() { |
| 556 _foundTests++; | 462 _foundTests++; |
| 557 } | 463 } |
| 558 | 464 |
| 559 void done(TestCase test) { | 465 void done(TestCase test) { |
| 560 if (test.unexpectedOutput) { | 466 if (test.unexpectedOutput) { |
| 561 _failedTests++; | 467 _failedTests++; |
| 562 } else { | 468 } else { |
| 563 _passedTests++; | 469 _passedTests++; |
| 564 } | 470 } |
| (...skipping 25 matching lines...) Expand all Loading... | |
| 590 } | 496 } |
| 591 print(''); | 497 print(''); |
| 592 } | 498 } |
| 593 | 499 |
| 594 void _printDoneProgress(TestCase test) => _printProgress(); | 500 void _printDoneProgress(TestCase test) => _printProgress(); |
| 595 | 501 |
| 596 void _printProgress(); | 502 void _printProgress(); |
| 597 } | 503 } |
| 598 | 504 |
| 599 class CompactProgressIndicator extends CompactIndicator { | 505 class CompactProgressIndicator extends CompactIndicator { |
| 600 Formatter _formatter; | 506 final Formatter _formatter; |
| 601 | 507 |
| 602 CompactProgressIndicator(DateTime startTime, this._formatter) | 508 CompactProgressIndicator(DateTime startTime, this._formatter) |
| 603 : super(startTime); | 509 : super(startTime); |
| 604 | 510 |
| 605 void _printProgress() { | 511 void _printProgress() { |
| 606 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString(); | 512 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString(); |
| 607 var progressPadded = _pad(_allTestsKnown ? percent : '--', 3); | 513 var progressPadded = (_allTestsKnown ? percent : '--').padLeft(3); |
| 608 var passedPadded = _pad(_passedTests.toString(), 5); | 514 var passedPadded = _passedTests.toString().padLeft(5); |
| 609 var failedPadded = _pad(_failedTests.toString(), 5); | 515 var failedPadded = _failedTests.toString().padLeft(5); |
| 610 Duration d = (new DateTime.now()).difference(_startTime); | 516 var elapsed = (new DateTime.now()).difference(_startTime); |
| 611 var progressLine = '\r[${_timeString(d)} | $progressPadded% | ' | 517 var progressLine = '\r[${_timeString(elapsed)} | $progressPadded% | ' |
| 612 '+${_formatter.passed(passedPadded)} | ' | 518 '+${_formatter.passed(passedPadded)} | ' |
| 613 '-${_formatter.failed(failedPadded)}]'; | 519 '-${_formatter.failed(failedPadded)}]'; |
| 614 stdout.write(progressLine); | 520 stdout.write(progressLine); |
| 615 } | 521 } |
| 616 } | 522 } |
| 617 | 523 |
| 618 class VerboseProgressIndicator extends ProgressIndicator { | 524 class VerboseProgressIndicator extends ProgressIndicator { |
| 619 VerboseProgressIndicator(DateTime startTime) : super(startTime); | 525 VerboseProgressIndicator(DateTime startTime) : super(startTime); |
| 620 | 526 |
| 621 void _printDoneProgress(TestCase test) { | 527 void _printDoneProgress(TestCase test) { |
| 622 var status = 'pass'; | 528 var status = 'pass'; |
| 623 if (test.unexpectedOutput) { | 529 if (test.unexpectedOutput) { |
| 624 status = 'fail'; | 530 status = 'fail'; |
| 625 } | 531 } |
| 626 print('Done ${test.configurationString} ${test.displayName}: $status'); | 532 print('Done ${test.configurationString} ${test.displayName}: $status'); |
| 627 } | 533 } |
| 628 } | 534 } |
| 629 | 535 |
| 630 class BuildbotProgressIndicator extends ProgressIndicator { | 536 class BuildbotProgressIndicator extends ProgressIndicator { |
| 631 static String stepName; | 537 static String stepName; |
| 632 var _failureSummary = <String>[]; | 538 final _failureSummary = <String>[]; |
| 633 | 539 |
| 634 BuildbotProgressIndicator(DateTime startTime) : super(startTime); | 540 BuildbotProgressIndicator(DateTime startTime) : super(startTime); |
| 635 | 541 |
| 636 void done(TestCase test) { | 542 void done(TestCase test) { |
| 637 super.done(test); | 543 super.done(test); |
| 638 if (test.unexpectedOutput) { | 544 if (test.unexpectedOutput) { |
| 639 _failureSummary.addAll(_buildFailureOutput(test)); | 545 _failureSummary.addAll(_buildFailureOutput(test)); |
| 640 } | 546 } |
| 641 } | 547 } |
| 642 | 548 |
| (...skipping 16 matching lines...) Expand all Loading... | |
| 659 } | 565 } |
| 660 for (String line in _failureSummary) { | 566 for (String line in _failureSummary) { |
| 661 print(line); | 567 print(line); |
| 662 } | 568 } |
| 663 print(''); | 569 print(''); |
| 664 } | 570 } |
| 665 print(_buildSummaryEnd(_failedTests)); | 571 print(_buildSummaryEnd(_failedTests)); |
| 666 } | 572 } |
| 667 } | 573 } |
| 668 | 574 |
| 669 EventListener progressIndicatorFromName( | 575 String _timeString(Duration duration) { |
| 670 String name, DateTime startTime, Formatter formatter) { | 576 var min = duration.inMinutes; |
| 671 switch (name) { | 577 var sec = duration.inSeconds % 60; |
| 672 case 'compact': | 578 return '${min.toString().padLeft(2, '0')}:${sec.toString().padLeft(2, '0')}'; |
| 673 return new CompactProgressIndicator(startTime, formatter); | 579 } |
| 674 case 'line': | 580 |
| 675 return new LineProgressIndicator(); | 581 List<String> _linesWithoutCarriageReturn(List<int> output) { |
| 676 case 'verbose': | 582 return decodeUtf8(output) |
| 677 return new VerboseProgressIndicator(startTime); | 583 .replaceAll('\r\n', '\n') |
| 678 case 'status': | 584 .replaceAll('\r', '\n') |
| 679 return new ProgressIndicator(startTime); | 585 .split('\n'); |
| 680 case 'buildbot': | 586 } |
| 681 return new BuildbotProgressIndicator(startTime); | 587 |
| 682 default: | 588 List<String> _buildFailureOutput(TestCase test, |
| 683 assert(false); | 589 [Formatter formatter = Formatter.normal]) { |
| 684 return null; | 590 var output = [ |
| 591 '', | |
| 592 formatter.failed('FAILED: ${test.configurationString}${test.displayName}') | |
|
ahe
2017/05/09 07:30:00
There's a space missing between ${test.configurati
Bob Nystrom
2017/05/09 17:32:36
Good catch, thanks!
https://codereview.chromium.o
| |
| 593 ]; | |
| 594 | |
| 595 var expected = new StringBuffer(); | |
| 596 expected.write('Expected: '); | |
| 597 for (var expectation in test.expectedOutcomes) { | |
| 598 expected.write('$expectation '); | |
| 599 } | |
| 600 | |
| 601 output.add(expected.toString()); | |
| 602 output.add('Actual: ${test.result}'); | |
| 603 if (!test.lastCommandOutput.hasTimedOut) { | |
| 604 if (test.commandOutputs.length != test.commands.length && | |
| 605 !test.expectCompileError) { | |
| 606 output.add('Unexpected compile-time error.'); | |
| 607 } else { | |
| 608 if (test.expectCompileError) { | |
| 609 output.add('Compile-time error expected.'); | |
| 610 } | |
| 611 if (test.hasRuntimeError) { | |
| 612 output.add('Runtime error expected.'); | |
| 613 } | |
| 614 if (test.configuration['checked'] && test.isNegativeIfChecked) { | |
| 615 output.add('Dynamic type error expected.'); | |
| 616 } | |
| 617 } | |
| 618 } | |
| 619 | |
| 620 for (var i = 0; i < test.commands.length; i++) { | |
| 621 var command = test.commands[i]; | |
| 622 var commandOutput = test.commandOutputs[command]; | |
| 623 if (commandOutput != null) { | |
| 624 output.add("CommandOutput[${command.displayName}]:"); | |
| 625 if (!commandOutput.diagnostics.isEmpty) { | |
| 626 String prefix = 'diagnostics:'; | |
| 627 for (var s in commandOutput.diagnostics) { | |
| 628 output.add('$prefix ${s}'); | |
| 629 prefix = ' '; | |
| 630 } | |
| 631 } | |
| 632 if (!commandOutput.stdout.isEmpty) { | |
| 633 output.add(''); | |
| 634 output.add('stdout:'); | |
| 635 output.addAll(_linesWithoutCarriageReturn(commandOutput.stdout)); | |
| 636 } | |
| 637 if (!commandOutput.stderr.isEmpty) { | |
| 638 output.add(''); | |
| 639 output.add('stderr:'); | |
| 640 output.addAll(_linesWithoutCarriageReturn(commandOutput.stderr)); | |
| 641 } | |
| 642 } | |
| 643 } | |
| 644 | |
| 645 if (test is BrowserTestCase) { | |
| 646 // Additional command for rerunning the steps locally after the fact. | |
| 647 var command = test.configuration["_servers_"].httpServerCommandLine(); | |
| 648 output.add(''); | |
| 649 output.add('To retest, run: $command'); | |
| 650 } | |
| 651 | |
| 652 for (var i = 0; i < test.commands.length; i++) { | |
| 653 var command = test.commands[i]; | |
| 654 var commandOutput = test.commandOutputs[command]; | |
| 655 output.add(''); | |
| 656 output.add('Command[${command.displayName}]: $command'); | |
| 657 if (commandOutput != null) { | |
| 658 output.add('Took ${commandOutput.time}'); | |
| 659 } else { | |
| 660 output.add('Did not run'); | |
| 661 } | |
| 662 } | |
| 663 | |
| 664 var arguments = ['python', 'tools/test.py']; | |
| 665 arguments.addAll(test.configuration['_reproducing_arguments_']); | |
| 666 arguments.add(test.displayName); | |
| 667 var testPyCommandline = arguments.map(escapeCommandLineArgument).join(' '); | |
| 668 | |
| 669 output.add(''); | |
| 670 output.add('Short reproduction command (experimental):'); | |
| 671 output.add(" $testPyCommandline"); | |
| 672 return output; | |
| 673 } | |
| 674 | |
| 675 String _buildSummaryEnd(int failedTests) { | |
| 676 if (failedTests == 0) { | |
| 677 return '\n===\n=== All tests succeeded\n===\n'; | |
| 678 } else { | |
| 679 var pluralSuffix = failedTests != 1 ? 's' : ''; | |
| 680 return '\n===\n=== ${failedTests} test$pluralSuffix failed\n===\n'; | |
| 685 } | 681 } |
| 686 } | 682 } |
| OLD | NEW |