Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(280)

Side by Side Diff: tools/testing/dart/test_progress.dart

Issue 2855073002: Small-scale clean ups in test.dart, mainly around test_progress. (Closed)
Patch Set: Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/testing/dart/test_options.dart ('k') | tools/testing/dart/test_suite.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library test_progress; 5 library test_progress;
6 6
7 import "dart:convert" show JSON;
7 import "dart:io"; 8 import "dart:io";
8 import "dart:io" as io; 9
9 import "dart:convert" show JSON;
10 import "path.dart"; 10 import "path.dart";
11 import "status_file_parser.dart"; 11 import "status_file_parser.dart";
12 import "summary_report.dart"; 12 import "summary_report.dart";
13 import "test_runner.dart"; 13 import "test_runner.dart";
14 import "test_suite.dart"; 14 import "test_suite.dart";
15 import "utils.dart"; 15 import "utils.dart";
16 16
17 String _pad(String s, int length) { 17 /// Controls how message strings are processed before being displayed.
18 StringBuffer buffer = new StringBuffer(); 18 class Formatter {
19 for (int i = s.length; i < length; i++) { 19 /// Messages are left as-is.
20 buffer.write(' '); 20 static const normal = const Formatter._();
21 } 21
22 buffer.write(s); 22 /// Messages are wrapped in ANSI escape codes to color them for display on a
23 return buffer.toString(); 23 /// terminal.
24 static const color = const _ColorFormatter();
25
26 const Formatter._();
27
28 /// Formats a success message.
29 String passed(String message) => message;
30
31 /// Formats a failure message.
32 String failed(String message) => message;
24 } 33 }
25 34
26 String _padTime(int time) { 35 class _ColorFormatter extends Formatter {
27 if (time == 0) { 36 static const _green = 32;
28 return '00'; 37 static const _red = 31;
29 } else if (time < 10) { 38 static const _escape = '\u001b';
30 return '0$time';
31 } else {
32 return '$time';
33 }
34 }
35 39
36 String _timeString(Duration d) { 40 const _ColorFormatter() : super._();
37 var min = d.inMinutes;
38 var sec = d.inSeconds % 60;
39 return '${_padTime(min)}:${_padTime(sec)}';
40 }
41 41
42 class Formatter { 42 String passed(String message) => _color(message, _green);
43 const Formatter(); 43 String failed(String message) => _color(message, _red);
44 String passed(msg) => msg;
45 String failed(msg) => msg;
46 }
47 44
48 class ColorFormatter extends Formatter { 45 static String _color(String message, int color) =>
49 static int BOLD = 1; 46 "$_escape[${color}m$message$_escape[0m";
50 static int GREEN = 32;
51 static int RED = 31;
52 static int NONE = 0;
53 static String ESCAPE = decodeUtf8([27]);
54
55 String passed(String msg) => _color(msg, GREEN);
56 String failed(String msg) => _color(msg, RED);
57
58 static String _color(String msg, int color) {
59 return "$ESCAPE[${color}m$msg$ESCAPE[0m";
60 }
61 }
62
63 List<String> _buildFailureOutput(TestCase test,
64 [Formatter formatter = const Formatter()]) {
65 List<String> getLinesWithoutCarriageReturn(List<int> output) {
66 return decodeUtf8(output)
67 .replaceAll('\r\n', '\n')
68 .replaceAll('\r', '\n')
69 .split('\n');
70 }
71
72 List<String> output = new List<String>();
73 output.add('');
74 output.add(formatter.failed('FAILED: ${test.configurationString}'
75 ' ${test.displayName}'));
76 StringBuffer expected = new StringBuffer();
77 expected.write('Expected: ');
78 for (var expectation in test.expectedOutcomes) {
79 expected.write('$expectation ');
80 }
81 output.add(expected.toString());
82 output.add('Actual: ${test.result}');
83 if (!test.lastCommandOutput.hasTimedOut) {
84 if (test.commandOutputs.length != test.commands.length &&
85 !test.expectCompileError) {
86 output.add('Unexpected compile-time error.');
87 } else {
88 if (test.expectCompileError) {
89 output.add('Compile-time error expected.');
90 }
91 if (test.hasRuntimeError) {
92 output.add('Runtime error expected.');
93 }
94 if (test.configuration['checked'] && test.isNegativeIfChecked) {
95 output.add('Dynamic type error expected.');
96 }
97 }
98 }
99 for (var i = 0; i < test.commands.length; i++) {
100 var command = test.commands[i];
101 var commandOutput = test.commandOutputs[command];
102 if (commandOutput != null) {
103 output.add("CommandOutput[${command.displayName}]:");
104 if (!commandOutput.diagnostics.isEmpty) {
105 String prefix = 'diagnostics:';
106 for (var s in commandOutput.diagnostics) {
107 output.add('$prefix ${s}');
108 prefix = ' ';
109 }
110 }
111 if (!commandOutput.stdout.isEmpty) {
112 output.add('');
113 output.add('stdout:');
114 output.addAll(getLinesWithoutCarriageReturn(commandOutput.stdout));
115 }
116 if (!commandOutput.stderr.isEmpty) {
117 output.add('');
118 output.add('stderr:');
119 output.addAll(getLinesWithoutCarriageReturn(commandOutput.stderr));
120 }
121 }
122 }
123 if (test is BrowserTestCase) {
124 // Additional command for rerunning the steps locally after the fact.
125 var command = test.configuration["_servers_"].httpServerCommandline();
126 output.add('');
127 output.add('To retest, run: $command');
128 }
129 for (var i = 0; i < test.commands.length; i++) {
130 var command = test.commands[i];
131 var commandOutput = test.commandOutputs[command];
132 output.add('');
133 output.add('Command[${command.displayName}]: $command');
134 if (commandOutput != null) {
135 output.add('Took ${commandOutput.time}');
136 } else {
137 output.add('Did not run');
138 }
139 }
140
141 var arguments = ['python', 'tools/test.py'];
142 arguments.addAll(test.configuration['_reproducing_arguments_']);
143 arguments.add(test.displayName);
144 var testPyCommandline = arguments.map(escapeCommandLineArgument).join(' ');
145
146 output.add('');
147 output.add('Short reproduction command (experimental):');
148 output.add(" $testPyCommandline");
149 return output;
150 }
151
152 String _buildSummaryEnd(int failedTests) {
153 if (failedTests == 0) {
154 return '\n===\n=== All tests succeeded\n===\n';
155 } else {
156 var pluralSuffix = failedTests != 1 ? 's' : '';
157 return '\n===\n=== ${failedTests} test$pluralSuffix failed\n===\n';
158 }
159 } 47 }
160 48
161 class EventListener { 49 class EventListener {
162 void testAdded() {} 50 void testAdded() {}
163 void done(TestCase test) {} 51 void done(TestCase test) {}
164 void allTestsKnown() {} 52 void allTestsKnown() {}
165 void allDone() {} 53 void allDone() {}
166 } 54 }
167 55
168 class ExitCodeSetter extends EventListener { 56 class ExitCodeSetter extends EventListener {
169 void done(TestCase test) { 57 void done(TestCase test) {
170 if (test.unexpectedOutput) { 58 if (test.unexpectedOutput) {
171 io.exitCode = 1; 59 exitCode = 1;
172 } 60 }
173 } 61 }
174 } 62 }
175 63
176 class IgnoredTestMonitor extends EventListener { 64 class IgnoredTestMonitor extends EventListener {
177 static final int maxIgnored = 5; 65 static final int maxIgnored = 5;
178 66
179 int countIgnored = 0; 67 int countIgnored = 0;
180 68
181 void done(TestCase test) { 69 void done(TestCase test) {
(...skipping 21 matching lines...) Expand all
203 if (test.isFlaky && test.result != Expectation.PASS) { 91 if (test.isFlaky && test.result != Expectation.PASS) {
204 var buf = new StringBuffer(); 92 var buf = new StringBuffer();
205 for (var l in _buildFailureOutput(test)) { 93 for (var l in _buildFailureOutput(test)) {
206 buf.write("$l\n"); 94 buf.write("$l\n");
207 } 95 }
208 _appendToFlakyFile(buf.toString()); 96 _appendToFlakyFile(buf.toString());
209 } 97 }
210 } 98 }
211 99
212 void _appendToFlakyFile(String msg) { 100 void _appendToFlakyFile(String msg) {
213 var file = new File(TestUtils.flakyFileName()); 101 var file = new File(TestUtils.flakyFileName);
214 var fd = file.openSync(mode: FileMode.APPEND); 102 var fd = file.openSync(mode: FileMode.APPEND);
215 fd.writeStringSync(msg); 103 fd.writeStringSync(msg);
216 fd.closeSync(); 104 fd.closeSync();
217 } 105 }
218 } 106 }
219 107
220 class TestOutcomeLogWriter extends EventListener { 108 class TestOutcomeLogWriter extends EventListener {
221 /* 109 /*
222 * The ".test-outcome.log" file contains one line for every executed test. 110 * The ".test-outcome.log" file contains one line for every executed test.
223 * Such a line is an encoded JSON data structure of the following form: 111 * Such a line is an encoded JSON data structure of the following form:
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after
300 }, 188 },
301 }); 189 });
302 } 190 }
303 191
304 void allDone() { 192 void allDone() {
305 if (_sink != null) _sink.close(); 193 if (_sink != null) _sink.close();
306 } 194 }
307 195
308 void _writeTestOutcomeRecord(Map record) { 196 void _writeTestOutcomeRecord(Map record) {
309 if (_sink == null) { 197 if (_sink == null) {
310 _sink = new File(TestUtils.testOutcomeFileName()) 198 _sink = new File(TestUtils.testOutcomeFileName)
311 .openWrite(mode: FileMode.APPEND); 199 .openWrite(mode: FileMode.APPEND);
312 } 200 }
313 _sink.write("${JSON.encode(record)}\n"); 201 _sink.write("${JSON.encode(record)}\n");
314 } 202 }
315 } 203 }
316 204
317 class UnexpectedCrashLogger extends EventListener { 205 class UnexpectedCrashLogger extends EventListener {
318 final archivedBinaries = <String, String>{}; 206 final archivedBinaries = <String, String>{};
319 207
320 void done(TestCase test) { 208 void done(TestCase test) {
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after
489 377
490 void done(TestCase test) { 378 void done(TestCase test) {
491 for (var commandOutput in test.commandOutputs.values) { 379 for (var commandOutput in test.commandOutputs.values) {
492 if (commandOutput.compilationSkipped) _skippedCompilations++; 380 if (commandOutput.compilationSkipped) _skippedCompilations++;
493 } 381 }
494 } 382 }
495 383
496 void allDone() { 384 void allDone() {
497 if (_skippedCompilations > 0) { 385 if (_skippedCompilations > 0) {
498 print('\n$_skippedCompilations compilations were skipped because ' 386 print('\n$_skippedCompilations compilations were skipped because '
499 'the previous output was already up to date\n'); 387 'the previous output was already up to date.\n');
500 } 388 }
501 } 389 }
502 } 390 }
503 391
504 class LineProgressIndicator extends EventListener { 392 class LineProgressIndicator extends EventListener {
505 void done(TestCase test) { 393 void done(TestCase test) {
506 var status = 'pass'; 394 var status = 'pass';
507 if (test.unexpectedOutput) { 395 if (test.unexpectedOutput) {
508 status = 'fail'; 396 status = 'fail';
509 } 397 }
510 print('Done ${test.configurationString} ${test.displayName}: $status'); 398 print('Done ${test.configurationString} ${test.displayName}: $status');
511 } 399 }
512 } 400 }
513 401
514 class TestFailurePrinter extends EventListener { 402 class TestFailurePrinter extends EventListener {
515 bool _printSummary; 403 final bool _printSummary;
516 var _formatter; 404 final Formatter _formatter;
517 var _failureSummary = <String>[]; 405 final _failureSummary = <String>[];
518 var _failedTests = 0; 406 int _failedTests = 0;
519 407
520 TestFailurePrinter(this._printSummary, [this._formatter = const Formatter()]); 408 TestFailurePrinter(this._printSummary, [this._formatter = Formatter.normal]);
521 409
522 void done(TestCase test) { 410 void done(TestCase test) {
523 if (test.unexpectedOutput) { 411 if (test.unexpectedOutput) {
524 _failedTests++; 412 _failedTests++;
525 var lines = _buildFailureOutput(test, _formatter); 413 var lines = _buildFailureOutput(test, _formatter);
526 for (var line in lines) { 414 for (var line in lines) {
527 print(line); 415 print(line);
528 } 416 }
529 print(''); 417 print('');
530 if (_printSummary) { 418 if (_printSummary) {
531 _failureSummary.addAll(lines); 419 _failureSummary.addAll(lines);
532 _failureSummary.add(''); 420 _failureSummary.add('');
533 } 421 }
534 } 422 }
535 } 423 }
536 424
537 void allDone() { 425 void allDone() {
538 if (_printSummary) { 426 if (_printSummary) {
539 if (!_failureSummary.isEmpty) { 427 if (!_failureSummary.isEmpty) {
540 print('\n=== Failure summary:\n'); 428 print('\n=== Failure summary:\n');
541 for (String line in _failureSummary) { 429 for (var line in _failureSummary) {
542 print(line); 430 print(line);
543 } 431 }
544 print(''); 432 print('');
545 433
546 print(_buildSummaryEnd(_failedTests)); 434 print(_buildSummaryEnd(_failedTests));
547 } 435 }
548 } 436 }
549 } 437 }
550 } 438 }
551 439
552 class ProgressIndicator extends EventListener { 440 class ProgressIndicator extends EventListener {
553 ProgressIndicator(this._startTime); 441 ProgressIndicator(this._startTime);
554 442
443 static EventListener fromName(
444 String name, DateTime startTime, Formatter formatter) {
445 switch (name) {
446 case 'compact':
447 return new CompactProgressIndicator(startTime, formatter);
448 case 'line':
449 return new LineProgressIndicator();
450 case 'verbose':
451 return new VerboseProgressIndicator(startTime);
452 case 'status':
453 return new ProgressIndicator(startTime);
454 case 'buildbot':
455 return new BuildbotProgressIndicator(startTime);
456 default:
457 throw new ArgumentError('Unknown progress indicator "$name".');
458 }
459 }
460
555 void testAdded() { 461 void testAdded() {
556 _foundTests++; 462 _foundTests++;
557 } 463 }
558 464
559 void done(TestCase test) { 465 void done(TestCase test) {
560 if (test.unexpectedOutput) { 466 if (test.unexpectedOutput) {
561 _failedTests++; 467 _failedTests++;
562 } else { 468 } else {
563 _passedTests++; 469 _passedTests++;
564 } 470 }
(...skipping 25 matching lines...) Expand all
590 } 496 }
591 print(''); 497 print('');
592 } 498 }
593 499
594 void _printDoneProgress(TestCase test) => _printProgress(); 500 void _printDoneProgress(TestCase test) => _printProgress();
595 501
596 void _printProgress(); 502 void _printProgress();
597 } 503 }
598 504
599 class CompactProgressIndicator extends CompactIndicator { 505 class CompactProgressIndicator extends CompactIndicator {
600 Formatter _formatter; 506 final Formatter _formatter;
601 507
602 CompactProgressIndicator(DateTime startTime, this._formatter) 508 CompactProgressIndicator(DateTime startTime, this._formatter)
603 : super(startTime); 509 : super(startTime);
604 510
605 void _printProgress() { 511 void _printProgress() {
606 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString(); 512 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString();
607 var progressPadded = _pad(_allTestsKnown ? percent : '--', 3); 513 var progressPadded = (_allTestsKnown ? percent : '--').padLeft(3);
608 var passedPadded = _pad(_passedTests.toString(), 5); 514 var passedPadded = _passedTests.toString().padLeft(5);
609 var failedPadded = _pad(_failedTests.toString(), 5); 515 var failedPadded = _failedTests.toString().padLeft(5);
610 Duration d = (new DateTime.now()).difference(_startTime); 516 var elapsed = (new DateTime.now()).difference(_startTime);
611 var progressLine = '\r[${_timeString(d)} | $progressPadded% | ' 517 var progressLine = '\r[${_timeString(elapsed)} | $progressPadded% | '
612 '+${_formatter.passed(passedPadded)} | ' 518 '+${_formatter.passed(passedPadded)} | '
613 '-${_formatter.failed(failedPadded)}]'; 519 '-${_formatter.failed(failedPadded)}]';
614 stdout.write(progressLine); 520 stdout.write(progressLine);
615 } 521 }
616 } 522 }
617 523
618 class VerboseProgressIndicator extends ProgressIndicator { 524 class VerboseProgressIndicator extends ProgressIndicator {
619 VerboseProgressIndicator(DateTime startTime) : super(startTime); 525 VerboseProgressIndicator(DateTime startTime) : super(startTime);
620 526
621 void _printDoneProgress(TestCase test) { 527 void _printDoneProgress(TestCase test) {
622 var status = 'pass'; 528 var status = 'pass';
623 if (test.unexpectedOutput) { 529 if (test.unexpectedOutput) {
624 status = 'fail'; 530 status = 'fail';
625 } 531 }
626 print('Done ${test.configurationString} ${test.displayName}: $status'); 532 print('Done ${test.configurationString} ${test.displayName}: $status');
627 } 533 }
628 } 534 }
629 535
630 class BuildbotProgressIndicator extends ProgressIndicator { 536 class BuildbotProgressIndicator extends ProgressIndicator {
631 static String stepName; 537 static String stepName;
632 var _failureSummary = <String>[]; 538 final _failureSummary = <String>[];
633 539
634 BuildbotProgressIndicator(DateTime startTime) : super(startTime); 540 BuildbotProgressIndicator(DateTime startTime) : super(startTime);
635 541
636 void done(TestCase test) { 542 void done(TestCase test) {
637 super.done(test); 543 super.done(test);
638 if (test.unexpectedOutput) { 544 if (test.unexpectedOutput) {
639 _failureSummary.addAll(_buildFailureOutput(test)); 545 _failureSummary.addAll(_buildFailureOutput(test));
640 } 546 }
641 } 547 }
642 548
(...skipping 16 matching lines...) Expand all
659 } 565 }
660 for (String line in _failureSummary) { 566 for (String line in _failureSummary) {
661 print(line); 567 print(line);
662 } 568 }
663 print(''); 569 print('');
664 } 570 }
665 print(_buildSummaryEnd(_failedTests)); 571 print(_buildSummaryEnd(_failedTests));
666 } 572 }
667 } 573 }
668 574
669 EventListener progressIndicatorFromName( 575 String _timeString(Duration duration) {
670 String name, DateTime startTime, Formatter formatter) { 576 var min = duration.inMinutes;
671 switch (name) { 577 var sec = duration.inSeconds % 60;
672 case 'compact': 578 return '${min.toString().padLeft(2, '0')}:${sec.toString().padLeft(2, '0')}';
673 return new CompactProgressIndicator(startTime, formatter); 579 }
674 case 'line': 580
675 return new LineProgressIndicator(); 581 List<String> _linesWithoutCarriageReturn(List<int> output) {
676 case 'verbose': 582 return decodeUtf8(output)
677 return new VerboseProgressIndicator(startTime); 583 .replaceAll('\r\n', '\n')
678 case 'status': 584 .replaceAll('\r', '\n')
679 return new ProgressIndicator(startTime); 585 .split('\n');
680 case 'buildbot': 586 }
681 return new BuildbotProgressIndicator(startTime); 587
682 default: 588 List<String> _buildFailureOutput(TestCase test,
683 assert(false); 589 [Formatter formatter = Formatter.normal]) {
684 return null; 590 var output = [
591 '',
592 formatter.failed('FAILED: ${test.configurationString}${test.displayName}')
ahe 2017/05/09 07:30:00 There's a space missing between ${test.configurati
Bob Nystrom 2017/05/09 17:32:36 Good catch, thanks! https://codereview.chromium.o
593 ];
594
595 var expected = new StringBuffer();
596 expected.write('Expected: ');
597 for (var expectation in test.expectedOutcomes) {
598 expected.write('$expectation ');
599 }
600
601 output.add(expected.toString());
602 output.add('Actual: ${test.result}');
603 if (!test.lastCommandOutput.hasTimedOut) {
604 if (test.commandOutputs.length != test.commands.length &&
605 !test.expectCompileError) {
606 output.add('Unexpected compile-time error.');
607 } else {
608 if (test.expectCompileError) {
609 output.add('Compile-time error expected.');
610 }
611 if (test.hasRuntimeError) {
612 output.add('Runtime error expected.');
613 }
614 if (test.configuration['checked'] && test.isNegativeIfChecked) {
615 output.add('Dynamic type error expected.');
616 }
617 }
618 }
619
620 for (var i = 0; i < test.commands.length; i++) {
621 var command = test.commands[i];
622 var commandOutput = test.commandOutputs[command];
623 if (commandOutput != null) {
624 output.add("CommandOutput[${command.displayName}]:");
625 if (!commandOutput.diagnostics.isEmpty) {
626 String prefix = 'diagnostics:';
627 for (var s in commandOutput.diagnostics) {
628 output.add('$prefix ${s}');
629 prefix = ' ';
630 }
631 }
632 if (!commandOutput.stdout.isEmpty) {
633 output.add('');
634 output.add('stdout:');
635 output.addAll(_linesWithoutCarriageReturn(commandOutput.stdout));
636 }
637 if (!commandOutput.stderr.isEmpty) {
638 output.add('');
639 output.add('stderr:');
640 output.addAll(_linesWithoutCarriageReturn(commandOutput.stderr));
641 }
642 }
643 }
644
645 if (test is BrowserTestCase) {
646 // Additional command for rerunning the steps locally after the fact.
647 var command = test.configuration["_servers_"].httpServerCommandLine();
648 output.add('');
649 output.add('To retest, run: $command');
650 }
651
652 for (var i = 0; i < test.commands.length; i++) {
653 var command = test.commands[i];
654 var commandOutput = test.commandOutputs[command];
655 output.add('');
656 output.add('Command[${command.displayName}]: $command');
657 if (commandOutput != null) {
658 output.add('Took ${commandOutput.time}');
659 } else {
660 output.add('Did not run');
661 }
662 }
663
664 var arguments = ['python', 'tools/test.py'];
665 arguments.addAll(test.configuration['_reproducing_arguments_']);
666 arguments.add(test.displayName);
667 var testPyCommandline = arguments.map(escapeCommandLineArgument).join(' ');
668
669 output.add('');
670 output.add('Short reproduction command (experimental):');
671 output.add(" $testPyCommandline");
672 return output;
673 }
674
675 String _buildSummaryEnd(int failedTests) {
676 if (failedTests == 0) {
677 return '\n===\n=== All tests succeeded\n===\n';
678 } else {
679 var pluralSuffix = failedTests != 1 ? 's' : '';
680 return '\n===\n=== ${failedTests} test$pluralSuffix failed\n===\n';
685 } 681 }
686 } 682 }
OLDNEW
« no previous file with comments | « tools/testing/dart/test_options.dart ('k') | tools/testing/dart/test_suite.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698