test.py: First step towards support of caching dart2js compilations across runtimes

tools/testing/dart/test_progress.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library test_progress;
 
 import "dart:io";
 import "dart:io" as io;
 import "http_server.dart" as http_server;
 import "status_file_parser.dart";
@@ skipping 59 matching lines @@
   List<String> output = new List<String>();
   output.add('');
   output.add(formatter.failed('FAILED: ${test.configurationString}'
                               ' ${test.displayName}'));
   StringBuffer expected = new StringBuffer();
   expected.write('Expected: ');
   for (var expectation in test.expectedOutcomes) {
     expected.write('$expectation ');
   }
   output.add(expected.toString());
-  output.add('Actual: ${test.lastCommandOutput.result}');
+  output.add('Actual: ${test.result}');
   if (!test.lastCommandOutput.hasTimedOut && test.info != null) {
-    if (test.lastCommandOutput.incomplete && !test.info.hasCompileError) {
+    if (test.commandOutputs.length != test.commands.length
+        && !test.info.hasCompileError) {
       output.add('Unexpected compile-time error.');
     } else {
       if (test.info.hasCompileError) {
         output.add('Compile-time error expected.');
       }
       if (test.info.hasRuntimeError) {
         output.add('Runtime error expected.');
       }
     }
   }
@@ skipping 30 matching lines @@
     // Additional command for rerunning the steps locally after the fact.
     var command =
       test.configuration["_servers_"].httpServerCommandline();
     output.add('');
     output.add('To retest, run:  $command');
   }
   for (var i = 0; i < test.commands.length; i++) {
     var command = test.commands[i];
     var commandOutput = test.commandOutputs[command];
     output.add('');
-    output.add('Command[$i]: $command');
+    output.add('Command[${command.displayName}}]: $command');

[ricow1, 2013/08/01 18:23:52]

extra }

[kustermann, 2013/08/05 07:29:17]

Done.

     if (commandOutput != null) {
       output.add('Took ${commandOutput.time}');
     } else {
       output.add('Did not run');
     }
   }
 
   var arguments = ['python', 'tools/test.py'];
   arguments.addAll(test.configuration['_reproducing_arguments_']);
   arguments.add(test.displayName);
@@ skipping 10 matching lines @@
       return '\n===\n=== All tests succeeded\n===\n';
     } else {
       var pluralSuffix = failedTests != 1 ? 's' : '';
       return '\n===\n=== ${failedTests} test$pluralSuffix failed\n===\n';
     }
 }
 
 
 class EventListener {
   void testAdded() { }
-  void start(TestCase test) { }
   void done(TestCase test) { }
   void allTestsKnown() { }
   void allDone() { }
 }
 
 class ExitCodeSetter extends EventListener {
   void done(TestCase test) {
-    if (test.lastCommandOutput.unexpectedOutput) {
+    if (test.unexpectedOutput) {
       io.exitCode = 1;
     }
   }
 }
 
 class FlakyLogWriter extends EventListener {
   void done(TestCase test) {
-    if (test.isFlaky && test.lastCommandOutput.result != PASS) {
+    if (test.isFlaky && test.result != PASS) {
       var buf = new StringBuffer();
       for (var l in _buildFailureOutput(test)) {
         buf.write("$l\n");
       }
       _appendToFlakyFile(buf.toString());
     }
   }
 
   void _appendToFlakyFile(String msg) {
     var file = new File(TestUtils.flakyFileName());
     var fd = file.openSync(mode: FileMode.APPEND);
     fd.writeStringSync(msg);
     fd.closeSync();
   }
 }
 
 class SummaryPrinter extends EventListener {
   void allTestsKnown() {
     if (SummaryReport.total > 0) {
       SummaryReport.printReport();
     }
   }
 }
 
 class TimingPrinter extends EventListener {
-  List<CommandOutput> _commandOutputs = <CommandOutput>[];
+  final _command2testCases = new Map<Command, List<TestCase>>();
+  final _commandOutputs = new Set<CommandOutput>();
   DateTime _startTime;
 
   TimingPrinter(this._startTime);
 
   void done(TestCase testCase) {
     for (var commandOutput in testCase.commandOutputs.values) {
+      var command = commandOutput.command;
       _commandOutputs.add(commandOutput);
+      _command2testCases.putIfAbsent(command, () => <TestCase>[]);
+      _command2testCases[command].add(testCase);
     }
   }
 
   void allDone() {
     Duration d = (new DateTime.now()).difference(_startTime);
     print('\n--- Total time: ${_timeString(d)} ---');
-    _commandOutputs.sort((a, b) {
+    var outputs = _commandOutputs.toList();
+    outputs.sort((a, b) {
       return b.time.inMilliseconds - a.time.inMilliseconds;
     });
-    for (int i = 0; i < 20 && i < _commandOutputs.length; i++) {
-      var commandOutput = _commandOutputs[i];
+    for (int i = 0; i < 20 && i < outputs.length; i++) {
+      var commandOutput = outputs[i];
       var command = commandOutput.command;
-      var testCase = commandOutput.testCase;
+      var testCases = _command2testCases[command];
       var duration = commandOutput.time;
-      var configuration = testCase.configurationString;
-      print('${commandOutput.time} - $configuration'
-            ' - ${testCase.displayName} (${command.displayName})');
+
+      var testCasesDescription = testCases.map((testCase) {
+        return "${testCase.configurationString}/${testCase.displayName}";
+      }).join(', ');
+
+      print('${commandOutput.time} - '
+            '${command.displayName} - '
+            '$testCasesDescription');
     }
   }
 }
 
 class StatusFileUpdatePrinter extends EventListener {
   var statusToConfigs = new Map<String, List<String>>();
   var _failureSummary = <String>[];
 
   void done(TestCase test) {
-    if (test.lastCommandOutput.unexpectedOutput) {
+    if (test.unexpectedOutput) {
       _printFailureOutput(test);
     }
   }
 
   void allDone() {
     _printFailureSummary();
   }
 
 
   void _printFailureOutput(TestCase test) {
-    String status = '${test.displayName}: ${test.lastCommandOutput.result}';
+    String status = '${test.displayName}: ${test.result}';
     List<String> configs =
         statusToConfigs.putIfAbsent(status, () => <String>[]);
     configs.add(test.configurationString);
     if (test.lastCommandOutput.hasTimedOut) {
       print('\n${test.displayName} timed out on ${test.configurationString}');
     }
   }
 
   String _extractRuntime(String configuration) {
     // Extract runtime from a configuration, for example,
@@ skipping 79 matching lines @@
                                 "in the system tempdir ('$systemTempDir')! "
                                 "Maybe left over directories?\n");
       }
     });
   }
 }
 
 class LineProgressIndicator extends EventListener {
   void done(TestCase test) {
     var status = 'pass';
-    if (test.lastCommandOutput.unexpectedOutput) {
+    if (test.unexpectedOutput) {
       status = 'fail';
     }
     print('Done ${test.configurationString} ${test.displayName}: $status');
   }
 }
 
 class TestFailurePrinter extends EventListener {
   bool _printSummary;
   var _formatter;
   var _failureSummary = <String>[];
   var _failedTests= 0;
 
   TestFailurePrinter(this._printSummary,
                      [this._formatter = const Formatter()]);
 
   void done(TestCase test) {
-    if (test.lastCommandOutput.unexpectedOutput) {
+    if (test.unexpectedOutput) {
       _failedTests++;
       var lines = _buildFailureOutput(test, _formatter);
       for (var line in lines) {
         print(line);
       }
       print('');
       if (_printSummary) {
         _failureSummary.addAll(lines);
         _failureSummary.add('');
       }
@@ skipping 14 matching lines @@
     }
   }
 }
 
 class ProgressIndicator extends EventListener {
   ProgressIndicator(this._startTime);
 
 
   void testAdded() { _foundTests++; }
 
-  void start(TestCase test) {
-    _printStartProgress(test);
-  }
-
   void done(TestCase test) {
-    if (test.lastCommandOutput.unexpectedOutput) {
+    if (test.unexpectedOutput) {
       _failedTests++;
     } else {
       _passedTests++;
     }
     _printDoneProgress(test);
   }
 
   void allTestsKnown() {
     _allTestsKnown = true;
   }
 
-  void _printStartProgress(TestCase test) {}
   void _printDoneProgress(TestCase test) {}
 
   int _completedTests() => _passedTests + _failedTests;
 
   int _foundTests = 0;
   int _passedTests = 0;
   int _failedTests = 0;
   bool _allTestsKnown = false;
   DateTime _startTime;
 }
 
 abstract class CompactIndicator extends ProgressIndicator {
   CompactIndicator(DateTime startTime)
       : super(startTime);
 
   void allDone() {
     if (_failedTests > 0) {
       // We may have printed many failure logs, so reprint the summary data.
       _printProgress();
     }
     print('');
   }
 
-  void _printStartProgress(TestCase test) => _printProgress();
   void _printDoneProgress(TestCase test) => _printProgress();
 
   void _printProgress();
 }
 
 
 class CompactProgressIndicator extends CompactIndicator {
   Formatter _formatter;
 
   CompactProgressIndicator(DateTime startTime, this._formatter)
@@ skipping 11 matching lines @@
         '-${_formatter.failed(failedPadded)}]';
     stdout.write(progressLine);
   }
 }
 
 
 class VerboseProgressIndicator extends ProgressIndicator {
   VerboseProgressIndicator(DateTime startTime)
       : super(startTime);
 
-  void _printStartProgress(TestCase test) {
-    print('Starting ${test.configurationString} ${test.displayName}...');
-  }
-
   void _printDoneProgress(TestCase test) {
     var status = 'pass';
-    if (test.lastCommandOutput.unexpectedOutput) {
+    if (test.unexpectedOutput) {
       status = 'fail';
     }
     print('Done ${test.configurationString} ${test.displayName}: $status');
   }
 }
 
 
 class BuildbotProgressIndicator extends ProgressIndicator {
   static String stepName;
   var _failureSummary = <String>[];
 
   BuildbotProgressIndicator(DateTime startTime) : super(startTime);
 
   void done(TestCase test) {
     super.done(test);
-    if (test.lastCommandOutput.unexpectedOutput) {
+    if (test.unexpectedOutput) {
       _failureSummary.addAll(_buildFailureOutput(test));
     }
   }
 
   void _printDoneProgress(TestCase test) {
     var status = 'pass';
-    if (test.lastCommandOutput.unexpectedOutput) {
+    if (test.unexpectedOutput) {
       status = 'fail';
     }
     var percent = ((_completedTests() / _foundTests) * 100).toInt().toString();
     print('Done ${test.configurationString} ${test.displayName}: $status');
     print('@@@STEP_CLEAR@@@');
     print('@@@STEP_TEXT@ $percent% +$_passedTests -$_failedTests @@@');
   }
 
   void allDone() {
     if (!_failureSummary.isEmpty) {
@@ skipping 23 matching lines @@
       return new VerboseProgressIndicator(startTime);
     case 'status':
       return new ProgressIndicator(startTime);
     case 'buildbot':
       return new BuildbotProgressIndicator(startTime);
     default:
       assert(false);
       break;
   }
 }