Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library test_progress; | 5 library test_progress; |
| 6 | 6 |
| 7 import "dart:io"; | 7 import "dart:io"; |
| 8 import "dart:io" as io; | 8 import "dart:io" as io; |
| 9 import "http_server.dart" as http_server; | 9 import "http_server.dart" as http_server; |
| 10 import "status_file_parser.dart"; | 10 import "status_file_parser.dart"; |
| 11 import "test_runner.dart"; | 11 import "test_runner.dart"; |
| 12 import "test_suite.dart"; | 12 import "test_suite.dart"; |
|
ricow1
2013/03/12 10:31:47
Martin: for future reference, stuff like this is e
| |
| 13 import "utils.dart"; | 13 import "utils.dart"; |
| 14 | 14 |
| 15 class ProgressIndicator { | 15 String _pad(String s, int length) { |
| 16 ProgressIndicator(this._startTime, this._printTiming) | 16 StringBuffer buffer = new StringBuffer(); |
| 17 : _tests = [], _failureSummary = []; | 17 for (int i = s.length; i < length; i++) { |
| 18 buffer.add(' '); | |
| 19 } | |
| 20 buffer.add(s); | |
| 21 return buffer.toString(); | |
| 22 } | |
| 18 | 23 |
| 19 factory ProgressIndicator.fromName(String name, | 24 String _padTime(int time) { |
| 20 Date startTime, | 25 if (time == 0) { |
| 21 bool printTiming) { | 26 return '00'; |
| 22 switch (name) { | 27 } else if (time < 10) { |
| 23 case 'compact': | 28 return '0$time'; |
| 24 return new CompactProgressIndicator(startTime, printTiming); | 29 } else { |
| 25 case 'color': | 30 return '$time'; |
| 26 return new ColorProgressIndicator(startTime, printTiming); | 31 } |
| 27 case 'line': | 32 } |
| 28 return new LineProgressIndicator(startTime, printTiming); | 33 |
| 29 case 'verbose': | 34 String _timeString(Duration d) { |
| 30 return new VerboseProgressIndicator(startTime, printTiming); | 35 var min = d.inMinutes; |
| 31 case 'silent': | 36 var sec = d.inSeconds % 60; |
| 32 return new SilentProgressIndicator(startTime, printTiming); | 37 return '${_padTime(min)}:${_padTime(sec)}'; |
| 33 case 'status': | 38 } |
| 34 return new StatusProgressIndicator(startTime, printTiming); | 39 |
| 35 case 'buildbot': | 40 class Formatter { |
| 36 return new BuildbotProgressIndicator(startTime, printTiming); | 41 const Formatter(); |
| 37 case 'diff': | 42 String passed(msg) => msg; |
| 38 return new DiffProgressIndicator(startTime, printTiming); | 43 String failed(msg) => msg; |
| 39 default: | 44 } |
| 40 assert(false); | 45 |
| 41 break; | 46 class ColorFormatter extends Formatter { |
| 47 static int BOLD = 1; | |
| 48 static int GREEN = 32; | |
| 49 static int RED = 31; | |
| 50 static int NONE = 0; | |
| 51 static String ESCAPE = decodeUtf8([27]); | |
| 52 | |
| 53 String passed(String msg) => _color(msg, GREEN); | |
| 54 String failed(String msg) => _color(msg, RED); | |
| 55 | |
| 56 static String _color(String msg, int color) { | |
| 57 return "$ESCAPE[${color}m$msg$ESCAPE[0m"; | |
| 58 } | |
| 59 } | |
| 60 | |
| 61 | |
| 62 List<String> _buildFailureOutput(TestCase test, | |
| 63 [Formatter formatter = const Formatter()]) { | |
| 64 List<String> output = new List<String>(); | |
| 65 output.add(''); | |
| 66 output.add(formatter.failed('FAILED: ${test.configurationString}' | |
| 67 ' ${test.displayName}')); | |
| 68 StringBuffer expected = new StringBuffer(); | |
| 69 expected.add('Expected: '); | |
| 70 for (var expectation in test.expectedOutcomes) { | |
| 71 expected.add('$expectation '); | |
| 72 } | |
| 73 output.add(expected.toString()); | |
| 74 output.add('Actual: ${test.lastCommandOutput.result}'); | |
| 75 if (!test.lastCommandOutput.hasTimedOut && test.info != null) { | |
| 76 if (test.lastCommandOutput.incomplete && !test.info.hasCompileError) { | |
| 77 output.add('Unexpected compile-time error.'); | |
| 78 } else { | |
| 79 if (test.info.hasCompileError) { | |
| 80 output.add('Compile-time error expected.'); | |
| 81 } | |
| 82 if (test.info.hasRuntimeError) { | |
| 83 output.add('Runtime error expected.'); | |
| 84 } | |
| 42 } | 85 } |
| 43 } | 86 } |
| 87 if (!test.lastCommandOutput.diagnostics.isEmpty) { | |
| 88 String prefix = 'diagnostics:'; | |
| 89 for (var s in test.lastCommandOutput.diagnostics) { | |
| 90 output.add('$prefix ${s}'); | |
| 91 prefix = ' '; | |
| 92 } | |
| 93 } | |
| 94 if (!test.lastCommandOutput.stdout.isEmpty) { | |
| 95 output.add(''); | |
| 96 output.add('stdout:'); | |
| 97 if (test.lastCommandOutput.command.isPixelTest) { | |
| 98 output.add('DRT pixel test failed! stdout is not printed because it ' | |
| 99 'contains binary data!'); | |
| 100 } else { | |
| 101 output.add(decodeUtf8(test.lastCommandOutput.stdout)); | |
| 102 } | |
| 103 } | |
| 104 if (!test.lastCommandOutput.stderr.isEmpty) { | |
| 105 output.add(''); | |
| 106 output.add('stderr:'); | |
| 107 output.add(decodeUtf8(test.lastCommandOutput.stderr)); | |
| 108 } | |
| 109 if (test is BrowserTestCase) { | |
| 110 // Additional command for rerunning the steps locally after the fact. | |
| 111 var command = | |
| 112 test.configuration["_servers_"].httpServerCommandline(); | |
| 113 output.add('To retest, run: $command'); | |
| 114 } | |
| 115 for (Command c in test.commands) { | |
| 116 output.add(''); | |
| 117 String message = (c == test.commands.last | |
| 118 ? "Command line" : "Compilation command"); | |
| 119 output.add('$message: $c'); | |
| 120 } | |
| 121 return output; | |
| 122 } | |
| 44 | 123 |
| 45 void testAdded() { _foundTests++; } | |
| 46 | 124 |
| 47 void start(TestCase test) { | 125 class EventListener { |
| 48 _printStartProgress(test); | 126 void testAdded() { } |
| 127 void start(TestCase test) { } | |
| 128 void done(TestCase test) { } | |
| 129 void allTestsKnown() { } | |
| 130 void allDone() { } | |
| 131 } | |
| 132 | |
| 133 class ExitCodeSetter extends EventListener { | |
| 134 bool _failingTest = false; | |
| 135 | |
| 136 void done(TestCase test) { | |
| 137 _failingTest = _failingTest || test.lastCommandOutput.unexpectedOutput; | |
|
ricow1
2013/03/12 10:31:47
can't we just set the exit code here?
kustermann
2013/03/12 13:59:16
Done.
| |
| 49 } | 138 } |
| 139 void allDone() { | |
| 140 if (_failingTest) { | |
| 141 io.exitCode = 1; | |
| 142 } | |
| 143 } | |
| 144 } | |
| 50 | 145 |
| 146 class FlakyLogWriter extends EventListener { | |
| 51 void done(TestCase test) { | 147 void done(TestCase test) { |
| 52 if (test.isFlaky && test.lastCommandOutput.result != PASS) { | 148 if (test.isFlaky && test.lastCommandOutput.result != PASS) { |
| 53 var buf = new StringBuffer(); | 149 var buf = new StringBuffer(); |
| 54 for (var l in _buildFailureOutput(test)) { | 150 for (var l in _buildFailureOutput(test)) { |
| 55 buf.add("$l\n"); | 151 buf.add("$l\n"); |
| 56 } | 152 } |
| 57 _appendToFlakyFile(buf.toString()); | 153 _appendToFlakyFile(buf.toString()); |
| 58 } | 154 } |
| 155 } | |
| 156 | |
| 157 void _appendToFlakyFile(String msg) { | |
| 158 var file = new File(TestUtils.flakyFileName()); | |
| 159 var fd = file.openSync(FileMode.APPEND); | |
| 160 fd.writeStringSync(msg); | |
| 161 fd.closeSync(); | |
| 162 } | |
| 163 } | |
| 164 | |
| 165 class SummaryPrinter extends EventListener { | |
| 166 void allTestsKnown() { | |
| 167 if (SummaryReport.total > 0) { | |
| 168 SummaryReport.printReport(); | |
| 169 } | |
| 170 } | |
| 171 } | |
| 172 | |
| 173 class TimingPrinter extends EventListener { | |
| 174 List<TestCase> _tests = <TestCase>[]; | |
| 175 Date _startTime; | |
| 176 | |
| 177 TimingPrinter(this._startTime); | |
|
ricow1
2013/03/12 10:31:47
can't we just set this to now?
kustermann
2013/03/12 13:59:16
I don't really care, I just kept it as it was befo
| |
| 178 | |
| 179 void done(TestCase testCase) { | |
| 180 _tests.add(testCase); | |
| 181 } | |
| 182 | |
| 183 void allDone() { | |
| 184 // TODO: We should take all the commands into account | |
|
ricow1
2013/03/12 10:31:47
Yes, this is probably very misleading for dart2js
kustermann
2013/03/12 13:59:16
We'll do it in another CL.
| |
| 185 Duration d = (new Date.now()).difference(_startTime); | |
| 186 print('\n--- Total time: ${_timeString(d)} ---'); | |
| 187 _tests.sort((a, b) { | |
| 188 Duration aDuration = a.lastCommandOutput.time; | |
| 189 Duration bDuration = b.lastCommandOutput.time; | |
| 190 return bDuration.inMilliseconds - aDuration.inMilliseconds; | |
| 191 }); | |
| 192 for (int i = 0; i < 20 && i < _tests.length; i++) { | |
| 193 var name = _tests[i].displayName; | |
| 194 var duration = _tests[i].lastCommandOutput.time; | |
| 195 var configuration = _tests[i].configurationString; | |
| 196 print('${duration} - $configuration $name'); | |
| 197 } | |
| 198 } | |
| 199 } | |
| 200 | |
| 201 class StatusFileUpdatePrinter extends EventListener { | |
| 202 var statusToConfigs = new Map<String, List<String>>(); | |
| 203 var _failureSummary = <String>[]; | |
| 204 | |
| 205 void done(TestCase test) { | |
| 206 if (test.lastCommandOutput.unexpectedOutput) { | |
| 207 _printFailureOutput(test); | |
| 208 } | |
| 209 } | |
| 210 | |
| 211 void allDone() { | |
| 212 _printFailureSummary(); | |
| 213 } | |
| 214 | |
| 215 | |
| 216 void _printFailureOutput(TestCase test) { | |
| 217 String status = '${test.displayName}: ${test.lastCommandOutput.result}'; | |
| 218 List<String> configs = | |
| 219 statusToConfigs.putIfAbsent(status, () => <String>[]); | |
| 220 configs.add(test.configurationString); | |
| 221 if (test.lastCommandOutput.hasTimedOut) { | |
| 222 print('\n${test.displayName} timed out on ${test.configurationString}'); | |
| 223 } | |
| 224 } | |
| 225 | |
| 226 String _extractRuntime(String configuration) { | |
| 227 // Extract runtime from a configuration, for example, | |
| 228 // 'none-vm-checked release_ia32'. | |
| 229 List<String> runtime = configuration.split(' ')[0].split('-'); | |
| 230 return '${runtime[0]}-${runtime[1]}'; | |
| 231 } | |
| 232 | |
| 233 void _printFailureSummary() { | |
| 234 var groupedStatuses = new Map<String, List<String>>(); | |
| 235 statusToConfigs.forEach((String status, List<String> configs) { | |
| 236 var runtimeToConfiguration = new Map<String, List<String>>(); | |
| 237 for (String config in configs) { | |
| 238 String runtime = _extractRuntime(config); | |
| 239 var runtimeConfigs = | |
| 240 runtimeToConfiguration.putIfAbsent(runtime, () => <String>[]); | |
| 241 runtimeConfigs.add(config); | |
| 242 } | |
| 243 runtimeToConfiguration.forEach((String runtime, | |
| 244 List<String> runtimeConfigs) { | |
| 245 runtimeConfigs.sort((a, b) => a.compareTo(b)); | |
| 246 List<String> statuses = | |
| 247 groupedStatuses.putIfAbsent('$runtime: $runtimeConfigs', | |
| 248 () => <String>[]); | |
| 249 statuses.add(status); | |
| 250 }); | |
| 251 }); | |
| 252 | |
| 253 print('\n\nNecessary status file updates:'); | |
| 254 groupedStatuses.forEach((String config, List<String> statuses) { | |
| 255 print(''); | |
| 256 print('$config:'); | |
| 257 statuses.sort((a, b) => a.compareTo(b)); | |
| 258 for (String status in statuses) { | |
| 259 print(' $status'); | |
| 260 } | |
| 261 }); | |
| 262 } | |
| 263 } | |
| 264 | |
| 265 class SkippedCompilationsPrinter extends EventListener { | |
| 266 int _skippedCompilations = 0; | |
| 267 | |
| 268 void done(TestCase test) { | |
| 59 for (var commandOutput in test.commandOutputs.values) { | 269 for (var commandOutput in test.commandOutputs.values) { |
| 60 if (commandOutput.compilationSkipped) | 270 if (commandOutput.compilationSkipped) |
| 61 _skippedCompilations++; | 271 _skippedCompilations++; |
| 62 } | 272 } |
| 63 | 273 } |
| 64 if (test.lastCommandOutput.unexpectedOutput) { | 274 |
| 65 _failedTests++; | 275 void allDone() { |
| 66 _printFailureOutput(test); | |
| 67 } else { | |
| 68 _passedTests++; | |
| 69 } | |
| 70 _printDoneProgress(test); | |
| 71 // If we need to print timing information we hold on to all completed | |
| 72 // tests. | |
| 73 if (_printTiming) _tests.add(test); | |
| 74 } | |
| 75 | |
| 76 void allTestsKnown() { | |
| 77 if (!_allTestsKnown) SummaryReport.printReport(); | |
| 78 _allTestsKnown = true; | |
| 79 } | |
| 80 | |
| 81 void _printSkippedCompilationInfo() { | |
| 82 if (_skippedCompilations > 0) { | 276 if (_skippedCompilations > 0) { |
| 83 print('\n$_skippedCompilations compilations were skipped because ' | 277 print('\n$_skippedCompilations compilations were skipped because ' |
| 84 'the previous output was already up to date\n'); | 278 'the previous output was already up to date\n'); |
| 85 } | 279 } |
| 86 } | 280 } |
| 87 | 281 } |
| 88 void _printTimingInformation() { | 282 |
| 89 if (_printTiming) { | 283 class LeftOverTempDirPrinter extends EventListener { |
| 90 // TODO: We should take all the commands into account | 284 final MIN_NUMBER_OF_TEMP_DIRS = 50; |
| 91 Duration d = (new Date.now()).difference(_startTime); | 285 |
| 92 print('\n--- Total time: ${_timeString(d)} ---'); | 286 Path _tempDir() { |
| 93 _tests.sort((a, b) { | 287 // Dir will be located in the system temporary directory. |
| 94 Duration aDuration = a.lastCommandOutput.time; | 288 var dir = new Directory('').createTempSync(); |
| 95 Duration bDuration = b.lastCommandOutput.time; | 289 var path = new Path(dir.path).directoryPath; |
| 96 return bDuration.inMilliseconds - aDuration.inMilliseconds; | 290 dir.deleteSync(); |
| 97 }); | 291 return path; |
| 98 for (int i = 0; i < 20 && i < _tests.length; i++) { | 292 } |
| 99 var name = _tests[i].displayName; | 293 |
| 100 var duration = _tests[i].lastCommandOutput.time; | 294 void allDone() { |
| 101 var configuration = _tests[i].configurationString; | 295 var tempDirs = []; |
| 102 print('${duration} - $configuration $name'); | 296 var systemTempDir = _tempDir(); |
| 103 } | 297 var lister = new Directory.fromPath(systemTempDir).list(); |
| 298 lister.onDir = (path) => tempDirs.add(path); | |
| 299 lister.onDone = (_) { | |
| 300 if (tempDirs.length > MIN_NUMBER_OF_TEMP_DIRS) { | |
| 301 DebugLogger.warning("There are more then $MIN_NUMBER_OF_TEMP_DIRS " | |
| 302 "directories in the system tempdir " | |
| 303 "('$systemTempDir')! Maybe left over directories?"); | |
| 304 } | |
| 305 }; | |
| 306 } | |
| 307 } | |
| 308 | |
| 309 class LineProgressIndicator extends EventListener { | |
| 310 void done(TestCase test) { | |
| 311 var status = 'pass'; | |
| 312 if (test.lastCommandOutput.unexpectedOutput) { | |
| 313 status = 'fail'; | |
| 314 } | |
| 315 print('Done ${test.configurationString} ${test.displayName}: $status'); | |
| 316 } | |
| 317 } | |
| 318 | |
| 319 class TestFailurePrinter extends EventListener { | |
| 320 var _failureSummary = <String>[]; | |
| 321 var _formatter; | |
| 322 | |
| 323 TestFailurePrinter([this._formatter = const Formatter()]); | |
| 324 | |
| 325 void done(TestCase test) { | |
| 326 if (test.lastCommandOutput.unexpectedOutput) { | |
| 327 _printFailureOutput(test); | |
| 104 } | 328 } |
| 105 } | 329 } |
| 106 | 330 |
| 107 void allDone() { | 331 void allDone() { |
| 108 _printFailureSummary(); | 332 _printFailureSummary(); |
| 109 _printStatus(); | 333 } |
| 110 _printSkippedCompilationInfo(); | |
| 111 _printTimingInformation(); | |
| 112 stdout.close(); | |
| 113 stderr.close(); | |
| 114 if (_failedTests > 0) { | |
| 115 io.exitCode = 1; | |
| 116 } | |
| 117 } | |
| 118 | |
| 119 void _printStartProgress(TestCase test) {} | |
| 120 void _printDoneProgress(TestCase test) {} | |
| 121 | |
| 122 String _pad(String s, int length) { | |
| 123 StringBuffer buffer = new StringBuffer(); | |
| 124 for (int i = s.length; i < length; i++) { | |
| 125 buffer.add(' '); | |
| 126 } | |
| 127 buffer.add(s); | |
| 128 return buffer.toString(); | |
| 129 } | |
| 130 | |
| 131 String _padTime(int time) { | |
| 132 if (time == 0) { | |
| 133 return '00'; | |
| 134 } else if (time < 10) { | |
| 135 return '0$time'; | |
| 136 } else { | |
| 137 return '$time'; | |
| 138 } | |
| 139 } | |
| 140 | |
| 141 String _timeString(Duration d) { | |
| 142 var min = d.inMinutes; | |
| 143 var sec = d.inSeconds % 60; | |
| 144 return '${_padTime(min)}:${_padTime(sec)}'; | |
| 145 } | |
| 146 | |
| 147 String _header(String header) => header; | |
| 148 | 334 |
| 149 void _printFailureOutput(TestCase test) { | 335 void _printFailureOutput(TestCase test) { |
| 150 var failureOutput = _buildFailureOutput(test); | 336 var failureOutput = _buildFailureOutput(test, _formatter); |
| 151 for (var line in failureOutput) { | 337 for (var line in failureOutput) { |
| 152 print(line); | 338 print(line); |
| 153 } | 339 } |
| 340 print(''); | |
| 154 _failureSummary.addAll(failureOutput); | 341 _failureSummary.addAll(failureOutput); |
| 155 } | 342 } |
| 156 | 343 |
| 157 List<String> _buildFailureOutput(TestCase test) { | |
| 158 List<String> output = new List<String>(); | |
| 159 output.add(''); | |
| 160 output.add(_header('FAILED: ${test.configurationString}' | |
| 161 ' ${test.displayName}')); | |
| 162 StringBuffer expected = new StringBuffer(); | |
| 163 expected.add('Expected: '); | |
| 164 for (var expectation in test.expectedOutcomes) { | |
| 165 expected.add('$expectation '); | |
| 166 } | |
| 167 output.add(expected.toString()); | |
| 168 output.add('Actual: ${test.lastCommandOutput.result}'); | |
| 169 if (!test.lastCommandOutput.hasTimedOut && test.info != null) { | |
| 170 if (test.lastCommandOutput.incomplete && !test.info.hasCompileError) { | |
| 171 output.add('Unexpected compile-time error.'); | |
| 172 } else { | |
| 173 if (test.info.hasCompileError) { | |
| 174 output.add('Compile-time error expected.'); | |
| 175 } | |
| 176 if (test.info.hasRuntimeError) { | |
| 177 output.add('Runtime error expected.'); | |
| 178 } | |
| 179 } | |
| 180 } | |
| 181 if (!test.lastCommandOutput.diagnostics.isEmpty) { | |
| 182 String prefix = 'diagnostics:'; | |
| 183 for (var s in test.lastCommandOutput.diagnostics) { | |
| 184 output.add('$prefix ${s}'); | |
| 185 prefix = ' '; | |
| 186 } | |
| 187 } | |
| 188 if (!test.lastCommandOutput.stdout.isEmpty) { | |
| 189 output.add(''); | |
| 190 output.add('stdout:'); | |
| 191 if (test.lastCommandOutput.command.isPixelTest) { | |
| 192 output.add('DRT pixel test failed! stdout is not printed because it ' | |
| 193 'contains binary data!'); | |
| 194 } else { | |
| 195 output.add(decodeUtf8(test.lastCommandOutput.stdout)); | |
| 196 } | |
| 197 } | |
| 198 if (!test.lastCommandOutput.stderr.isEmpty) { | |
| 199 output.add(''); | |
| 200 output.add('stderr:'); | |
| 201 output.add(decodeUtf8(test.lastCommandOutput.stderr)); | |
| 202 } | |
| 203 if (test is BrowserTestCase) { | |
| 204 // Additional command for rerunning the steps locally after the fact. | |
| 205 var command = | |
| 206 test.configuration["_servers_"].httpServerCommandline(); | |
| 207 output.add('To retest, run: $command'); | |
| 208 } | |
| 209 for (Command c in test.commands) { | |
| 210 output.add(''); | |
| 211 String message = (c == test.commands.last | |
| 212 ? "Command line" : "Compilation command"); | |
| 213 output.add('$message: $c'); | |
| 214 } | |
| 215 return output; | |
| 216 } | |
| 217 | |
| 218 void _printFailureSummary() { | 344 void _printFailureSummary() { |
| 219 for (String line in _failureSummary) { | 345 for (String line in _failureSummary) { |
| 220 print(line); | 346 print(line); |
| 221 } | 347 } |
| 222 print(''); | 348 print(''); |
| 223 } | 349 } |
| 350 } | |
| 351 | |
| 352 class ProgressIndicator extends EventListener { | |
| 353 ProgressIndicator(this._startTime); | |
| 354 | |
| 355 factory ProgressIndicator.fromName(String name, | |
| 356 Date startTime, | |
| 357 Formatter formatter) { | |
| 358 switch (name) { | |
| 359 case 'compact': | |
| 360 return new CompactProgressIndicator(startTime, formatter); | |
| 361 case 'line': | |
| 362 return new LineProgressIndicator(); | |
| 363 case 'verbose': | |
| 364 return new VerboseProgressIndicator(startTime); | |
| 365 case 'status': | |
| 366 return new ProgressIndicator(startTime); | |
| 367 case 'buildbot': | |
| 368 return new BuildbotProgressIndicator(startTime); | |
| 369 default: | |
| 370 assert(false); | |
| 371 break; | |
| 372 } | |
| 373 } | |
| 374 | |
| 375 void testAdded() { _foundTests++; } | |
| 376 | |
| 377 void start(TestCase test) { | |
| 378 _printStartProgress(test); | |
| 379 } | |
| 380 | |
| 381 void done(TestCase test) { | |
| 382 if (test.lastCommandOutput.unexpectedOutput) { | |
| 383 _failedTests++; | |
| 384 } else { | |
| 385 _passedTests++; | |
| 386 } | |
| 387 _printDoneProgress(test); | |
| 388 } | |
| 389 | |
| 390 void allTestsKnown() { | |
| 391 _allTestsKnown = true; | |
| 392 } | |
| 393 | |
| 394 void allDone() { | |
| 395 _printStatus(); | |
| 396 stdout.close(); | |
| 397 stderr.close(); | |
| 398 } | |
| 399 | |
| 400 void _printStartProgress(TestCase test) {} | |
| 401 void _printDoneProgress(TestCase test) {} | |
| 224 | 402 |
| 225 void _printStatus() { | 403 void _printStatus() { |
| 226 if (_failedTests == 0) { | 404 if (_failedTests == 0) { |
| 227 print('\n==='); | 405 print('\n==='); |
| 228 print('=== All tests succeeded'); | 406 print('=== All tests succeeded'); |
| 229 print('===\n'); | 407 print('===\n'); |
| 230 } else { | 408 } else { |
| 231 var pluralSuffix = _failedTests != 1 ? 's' : ''; | 409 var pluralSuffix = _failedTests != 1 ? 's' : ''; |
| 232 print('\n==='); | 410 print('\n==='); |
| 233 print('=== ${_failedTests} test$pluralSuffix failed'); | 411 print('=== ${_failedTests} test$pluralSuffix failed'); |
| 234 print('===\n'); | 412 print('===\n'); |
| 235 } | 413 } |
| 236 } | 414 } |
| 237 | 415 |
| 238 void _appendToFlakyFile(String msg) { | |
| 239 var file = new File(TestUtils.flakyFileName()); | |
| 240 var fd = file.openSync(FileMode.APPEND); | |
| 241 fd.writeStringSync(msg); | |
| 242 fd.closeSync(); | |
| 243 } | |
| 244 | |
| 245 int get numFailedTests => _failedTests; | 416 int get numFailedTests => _failedTests; |
| 246 | 417 |
| 247 int _completedTests() => _passedTests + _failedTests; | 418 int _completedTests() => _passedTests + _failedTests; |
| 248 | 419 |
| 249 int _foundTests = 0; | 420 int _foundTests = 0; |
| 250 int _passedTests = 0; | 421 int _passedTests = 0; |
| 251 int _failedTests = 0; | 422 int _failedTests = 0; |
| 252 int _skippedCompilations = 0; | |
| 253 bool _allTestsKnown = false; | 423 bool _allTestsKnown = false; |
| 254 Date _startTime; | 424 Date _startTime; |
| 255 bool _printTiming; | |
| 256 List<TestCase> _tests; | |
| 257 List<String> _failureSummary; | |
| 258 } | |
| 259 | |
| 260 | |
| 261 class SilentProgressIndicator extends ProgressIndicator { | |
| 262 SilentProgressIndicator(Date startTime, bool printTiming) | |
| 263 : super(startTime, printTiming); | |
| 264 void testAdded() { } | |
| 265 void start(TestCase test) { } | |
| 266 void done(TestCase test) { } | |
| 267 void _printStartProgress(TestCase test) { } | |
| 268 void _printDoneProgress(TestCase test) { } | |
| 269 void allTestsKnown() { } | |
| 270 void allDone() { } | |
| 271 } | 425 } |
| 272 | 426 |
| 273 abstract class CompactIndicator extends ProgressIndicator { | 427 abstract class CompactIndicator extends ProgressIndicator { |
| 274 CompactIndicator(Date startTime, bool printTiming) | 428 CompactIndicator(Date startTime) |
| 275 : super(startTime, printTiming); | 429 : super(startTime); |
| 276 | 430 |
| 277 void allDone() { | 431 void allDone() { |
| 278 stdout.write('\n'.charCodes); | 432 stdout.write('\n'.charCodes); |
| 279 _printFailureSummary(); | |
| 280 _printSkippedCompilationInfo(); | |
| 281 _printTimingInformation(); | |
| 282 if (_failedTests > 0) { | 433 if (_failedTests > 0) { |
| 283 // We may have printed many failure logs, so reprint the summary data. | 434 // We may have printed many failure logs, so reprint the summary data. |
| 284 _printProgress(); | 435 _printProgress(); |
| 285 print(''); | 436 print(''); |
| 286 } | 437 } |
| 287 stdout.close(); | 438 stdout.close(); |
| 288 stderr.close(); | 439 stderr.close(); |
| 289 if (_failedTests > 0) { | |
| 290 io.exitCode = 1; | |
| 291 } | |
| 292 } | |
| 293 | |
| 294 void allTestsKnown() { | |
| 295 if (!_allTestsKnown && SummaryReport.total > 0) { | |
| 296 // Clear progress indicator before printing summary report. | |
| 297 stdout.write( | |
| 298 '\r \r'.charCodes); | |
| 299 SummaryReport.printReport(); | |
| 300 } | |
| 301 _allTestsKnown = true; | |
| 302 } | 440 } |
| 303 | 441 |
| 304 void _printStartProgress(TestCase test) => _printProgress(); | 442 void _printStartProgress(TestCase test) => _printProgress(); |
| 305 void _printDoneProgress(TestCase test) => _printProgress(); | 443 void _printDoneProgress(TestCase test) => _printProgress(); |
| 306 | 444 |
| 307 void _printProgress(); | 445 void _printProgress(); |
| 308 } | 446 } |
| 309 | 447 |
| 310 | 448 |
| 311 class CompactProgressIndicator extends CompactIndicator { | 449 class CompactProgressIndicator extends CompactIndicator { |
| 312 CompactProgressIndicator(Date startTime, bool printTiming) | 450 Formatter _formatter; |
| 313 : super(startTime, printTiming); | 451 |
| 452 CompactProgressIndicator(Date startTime, this._formatter) | |
| 453 : super(startTime); | |
| 314 | 454 |
| 315 void _printProgress() { | 455 void _printProgress() { |
| 316 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString(); | 456 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString(); |
| 317 var progressPadded = _pad(_allTestsKnown ? percent : '--', 3); | 457 var progressPadded = _pad(_allTestsKnown ? percent : '--', 3); |
| 318 var passedPadded = _pad(_passedTests.toString(), 5); | 458 var passedPadded = _pad(_passedTests.toString(), 5); |
| 319 var failedPadded = _pad(_failedTests.toString(), 5); | 459 var failedPadded = _pad(_failedTests.toString(), 5); |
| 320 Duration d = (new Date.now()).difference(_startTime); | 460 Duration d = (new Date.now()).difference(_startTime); |
| 321 var progressLine = | 461 var progressLine = |
| 322 '\r[${_timeString(d)} | $progressPadded% | ' | 462 '\r[${_timeString(d)} | $progressPadded% | ' |
| 323 '+$passedPadded | -$failedPadded]'; | 463 '+${_formatter.passed(passedPadded)} | ' |
| 464 '-${_formatter.failed(failedPadded)}]'; | |
| 324 stdout.write(progressLine.charCodes); | 465 stdout.write(progressLine.charCodes); |
| 325 } | 466 } |
| 326 } | 467 } |
| 327 | 468 |
| 328 | 469 |
| 329 class ColorProgressIndicator extends CompactIndicator { | |
| 330 ColorProgressIndicator(Date startTime, bool printTiming) | |
| 331 : super(startTime, printTiming); | |
| 332 | |
| 333 static int BOLD = 1; | |
| 334 static int GREEN = 32; | |
| 335 static int RED = 31; | |
| 336 static int NONE = 0; | |
| 337 | |
| 338 addColorWrapped(List<int> codes, String string, int color) { | |
| 339 codes.add(27); | |
| 340 codes.addAll('[${color}m'.charCodes); | |
| 341 codes.addAll(encodeUtf8(string)); | |
| 342 codes.add(27); | |
| 343 codes.addAll('[0m'.charCodes); | |
| 344 } | |
| 345 | |
| 346 void _printProgress() { | |
| 347 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString(); | |
| 348 var progressPadded = _pad(_allTestsKnown ? percent : '--', 3); | |
| 349 var passedPadded = _pad(_passedTests.toString(), 5); | |
| 350 var failedPadded = _pad(_failedTests.toString(), 5); | |
| 351 Duration d = (new Date.now()).difference(_startTime); | |
| 352 var progressLine = []; | |
| 353 progressLine.addAll('\r[${_timeString(d)} | $progressPadded% | '.charCodes); | |
| 354 addColorWrapped(progressLine, '+$passedPadded ', GREEN); | |
| 355 progressLine.addAll('| '.charCodes); | |
| 356 var failedColor = (_failedTests != 0) ? RED : NONE; | |
| 357 addColorWrapped(progressLine, '-$failedPadded', failedColor); | |
| 358 progressLine.addAll(']'.charCodes); | |
| 359 stdout.write(progressLine); | |
| 360 } | |
| 361 | |
| 362 String _header(String header) { | |
| 363 var result = []; | |
| 364 addColorWrapped(result, header, BOLD); | |
| 365 return decodeUtf8(result); | |
| 366 } | |
| 367 } | |
| 368 | |
| 369 | |
| 370 class LineProgressIndicator extends ProgressIndicator { | |
| 371 LineProgressIndicator(Date startTime, bool printTiming) | |
| 372 : super(startTime, printTiming); | |
| 373 | |
| 374 void _printStartProgress(TestCase test) { | |
| 375 } | |
| 376 | |
| 377 void _printDoneProgress(TestCase test) { | |
| 378 var status = 'pass'; | |
| 379 if (test.lastCommandOutput.unexpectedOutput) { | |
| 380 status = 'fail'; | |
| 381 } | |
| 382 print('Done ${test.configurationString} ${test.displayName}: $status'); | |
| 383 } | |
| 384 } | |
| 385 | |
| 386 | |
| 387 class VerboseProgressIndicator extends ProgressIndicator { | 470 class VerboseProgressIndicator extends ProgressIndicator { |
| 388 VerboseProgressIndicator(Date startTime, bool printTiming) | 471 VerboseProgressIndicator(Date startTime) |
| 389 : super(startTime, printTiming); | 472 : super(startTime); |
| 390 | 473 |
| 391 void _printStartProgress(TestCase test) { | 474 void _printStartProgress(TestCase test) { |
| 392 print('Starting ${test.configurationString} ${test.displayName}...'); | 475 print('Starting ${test.configurationString} ${test.displayName}...'); |
| 393 } | 476 } |
| 394 | 477 |
| 395 void _printDoneProgress(TestCase test) { | 478 void _printDoneProgress(TestCase test) { |
| 396 var status = 'pass'; | 479 var status = 'pass'; |
| 397 if (test.lastCommandOutput.unexpectedOutput) { | 480 if (test.lastCommandOutput.unexpectedOutput) { |
| 398 status = 'fail'; | 481 status = 'fail'; |
| 399 } | 482 } |
| 400 print('Done ${test.configurationString} ${test.displayName}: $status'); | 483 print('Done ${test.configurationString} ${test.displayName}: $status'); |
| 401 } | 484 } |
| 402 } | 485 } |
| 403 | 486 |
| 404 | 487 |
| 405 class StatusProgressIndicator extends ProgressIndicator { | |
| 406 StatusProgressIndicator(Date startTime, bool printTiming) | |
| 407 : super(startTime, printTiming); | |
| 408 | |
| 409 void _printStartProgress(TestCase test) { | |
| 410 } | |
| 411 | |
| 412 void _printDoneProgress(TestCase test) { | |
| 413 } | |
| 414 } | |
| 415 | |
| 416 | |
| 417 class BuildbotProgressIndicator extends ProgressIndicator { | 488 class BuildbotProgressIndicator extends ProgressIndicator { |
| 418 static String stepName; | 489 static String stepName; |
| 419 | 490 |
| 420 BuildbotProgressIndicator(Date startTime, bool printTiming) | 491 BuildbotProgressIndicator(Date startTime) : super(startTime); |
| 421 : super(startTime, printTiming); | |
| 422 | 492 |
| 423 void _printStartProgress(TestCase test) { | 493 void _printStartProgress(TestCase test) { } |
|
ricow1
2013/03/12 10:31:47
we don't need this
kustermann
2013/03/12 13:59:16
Done.
| |
| 424 } | |
| 425 | 494 |
| 426 void _printDoneProgress(TestCase test) { | 495 void _printDoneProgress(TestCase test) { |
| 427 var status = 'pass'; | 496 var status = 'pass'; |
| 428 if (test.lastCommandOutput.unexpectedOutput) { | 497 if (test.lastCommandOutput.unexpectedOutput) { |
| 429 status = 'fail'; | 498 status = 'fail'; |
| 430 } | 499 } |
| 431 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString(); | 500 var percent = ((_completedTests() / _foundTests) * 100).toInt().toString(); |
| 432 print('Done ${test.configurationString} ${test.displayName}: $status'); | 501 print('Done ${test.configurationString} ${test.displayName}: $status'); |
| 433 print('@@@STEP_CLEAR@@@'); | 502 print('@@@STEP_CLEAR@@@'); |
| 434 print('@@@STEP_TEXT@ $percent% +$_passedTests -$_failedTests @@@'); | 503 print('@@@STEP_TEXT@ $percent% +$_passedTests -$_failedTests @@@'); |
| 435 } | 504 } |
| 436 | 505 |
| 437 void _printFailureSummary() { | 506 void allDone() { |
| 438 if (!_failureSummary.isEmpty && stepName != null) { | 507 if (_failedTests > 0) { |
| 439 print('@@@STEP_FAILURE@@@'); | 508 print('@@@STEP_FAILURE@@@'); |
| 440 print('@@@BUILD_STEP $stepName failures@@@'); | 509 print('@@@BUILD_STEP $stepName failures@@@'); |
| 441 } | 510 } |
| 442 super._printFailureSummary(); | |
| 443 } | 511 } |
| 444 } | 512 } |
| 445 | 513 |
| 446 class DiffProgressIndicator extends ColorProgressIndicator { | |
| 447 Map<String, List<String>> statusToConfigs = new Map<String, List<String>>(); | |
| 448 | |
| 449 DiffProgressIndicator(Date startTime, bool printTiming) | |
| 450 : super(startTime, printTiming); | |
| 451 | |
| 452 void _printFailureOutput(TestCase test) { | |
| 453 String status = '${test.displayName}: ${test.lastCommandOutput.result}'; | |
| 454 List<String> configs = | |
| 455 statusToConfigs.putIfAbsent(status, () => <String>[]); | |
| 456 configs.add(test.configurationString); | |
| 457 if (test.lastCommandOutput.hasTimedOut) { | |
| 458 print('\n${test.displayName} timed out on ${test.configurationString}'); | |
| 459 } | |
| 460 } | |
| 461 | |
| 462 String _extractRuntime(String configuration) { | |
| 463 // Extract runtime from a configuration, for example, | |
| 464 // 'none-vm-checked release_ia32'. | |
| 465 List<String> runtime = configuration.split(' ')[0].split('-'); | |
| 466 return '${runtime[0]}-${runtime[1]}'; | |
| 467 } | |
| 468 | |
| 469 void _printFailureSummary() { | |
| 470 Map<String, List<String>> groupedStatuses = new Map<String, List<String>>(); | |
| 471 statusToConfigs.forEach((String status, List<String> configs) { | |
| 472 Map<String, List<String>> runtimeToConfiguration = | |
| 473 new Map<String, List<String>>(); | |
| 474 for (String config in configs) { | |
| 475 String runtime = _extractRuntime(config); | |
| 476 List<String> runtimeConfigs = | |
| 477 runtimeToConfiguration.putIfAbsent(runtime, () => <String>[]); | |
| 478 runtimeConfigs.add(config); | |
| 479 } | |
| 480 runtimeToConfiguration.forEach((String runtime, | |
| 481 List<String> runtimeConfigs) { | |
| 482 runtimeConfigs.sort((a, b) => a.compareTo(b)); | |
| 483 List<String> statuses = | |
| 484 groupedStatuses.putIfAbsent('$runtime: $runtimeConfigs', | |
| 485 () => <String>[]); | |
| 486 statuses.add(status); | |
| 487 }); | |
| 488 }); | |
| 489 groupedStatuses.forEach((String config, List<String> statuses) { | |
| 490 print(''); | |
| 491 print(''); | |
| 492 print('$config:'); | |
| 493 statuses.sort((a, b) => a.compareTo(b)); | |
| 494 for (String status in statuses) { | |
| 495 print(' $status'); | |
| 496 } | |
| 497 }); | |
| 498 _printStatus(); | |
| 499 } | |
| 500 } | |
| OLD | NEW |