Outputing comma-separated report with percentages

tools/testing/dart/status_reporter.dart

Index: tools/testing/dart/status_reporter.dart
diff --git a/tools/testing/dart/status_reporter.dart b/tools/testing/dart/status_reporter.dart
new file mode 100644
index 0000000000000000000000000000000000000000..50ea48dbb6cb5ac2f320c57abd60a08e2e189f92
--- /dev/null
+++ b/tools/testing/dart/status_reporter.dart
@@ -0,0 +1,183 @@
+// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:convert';
+
+List<Map> LINUX_COMBINATIONS = [
+    {
+      'runtimes' : ['none'],
+      'modes' : ['release'],
+      'archs' : ['ia32'],
+      'compiler' : 'dartanalyzer'
+    },
+    {
+      'runtimes' : ['vm'],
+      'modes' : ['debug', 'release'],
+      'archs' : ['ia32', 'x64', 'simarm', 'simmips'],
+      'compiler' : 'none'
+    },
+    {
+      'runtimes' : ['vm'],
+      'modes' : ['release'],
+      'archs' : ['ia32', 'x64'],
+      'compiler' : 'dart2dart'
+    },
+    {
+      'runtimes' : ['d8', 'jsshell', 'chrome', 'ff'],
+      'modes' : ['release'],
+      'archs' : ['ia32'],
+      'compiler' : 'dart2js'
+    },
+    {
+      'runtimes' : ['dartium'],
+      'modes' : ['release', 'debug'],
+      'archs' : ['ia32'],
+      'compiler' : 'none'
+    },
+];
+
+List<Map> MACOS_COMBINATIONS = [
+    {
+      'runtimes' : ['vm'],
+      'modes' : ['debug', 'release'],
+      'archs' : ['ia32', 'x64'],
+      'compiler' : 'none'
+    },
+    {
+      'runtimes' : ['safari', 'safarimobilesim'],
+      'modes' : ['release'],
+      'archs' : ['ia32'],
+      'compiler' : 'dart2js'
+    },
+    {
+      'runtimes' : ['dartium'],
+      'modes' : ['release', 'debug'],
+      'archs' : ['ia32'],
+      'compiler' : 'none'
+    },
+];
+
+List<Map> WINDOWS_COMBINATIONS = [
+    {
+      'runtimes' : ['vm'],
+      'modes' : ['debug', 'release'],
+      'archs' : ['ia32', 'x64'],
+      'compiler' : 'none'
+    },
+    {
+      'runtimes' : ['chrome', 'ff', 'ie11', 'ie10'],
+      'modes' : ['release'],
+      'archs' : ['ia32'],
+      'compiler' : 'dart2js'
+    },
+    {
+      'runtimes' : ['dartium'],
+      'modes' : ['release', 'debug'],
+      'archs' : ['ia32'],
+      'compiler' : 'none'
+    },
+];
+
+Map<String, List<Map>> COMBINATIONS = {
+  'linux' : LINUX_COMBINATIONS,
+  'windows' : WINDOWS_COMBINATIONS,
+  'macos' : MACOS_COMBINATIONS
+};
+
+List<Map> getCombinations() {
+  return COMBINATIONS[Platform.operatingSystem];
+}
+
+void ensureBuild() {
+  print('Building - please be patient we build a lot of crap');

[ricow1, 2015/01/13 08:03:00]

I know I wrote this, but maybe change the wording

[kevmoo, 2015/01/13 19:27:35]

Done.

+  var archs = Platform.operatingSystem == 'linux'
+    ? '-aia32,x64,simarm,simmips'
+    : '-aia32,x64';
+  var result = Process.runSync('python',  ['tools/build.py', '-mrelease,debug',

[ricow1, 2015/01/13 08:03:01]

add comment stating:
// We build runtime to be able to list cc tests

[kevmoo, 2015/01/13 19:27:35]

Done.

+                                           archs,
+                                           'create_sdk', 'runtime']);
+  if (result.exitCode != 0) {
+    print('ERROR');

[ricow1, 2015/01/13 08:03:01]

remove this line

[kevmoo, 2015/01/13 19:27:35]

Done.

+    print(result.stdout);
+    print(result.stderr);

[ricow1, 2015/01/13 08:03:01]

trow after this, we don't want to pretend we can run the tests if we can't build

[kevmoo, 2015/01/13 19:27:35]

Done.

+  }
+  print('done building');

[ricow1, 2015/01/13 08:03:00]

s/done/Done

[kevmoo, 2015/01/13 19:27:35]

Done.

+}
+
+void sanityCheck(String output) {
+  LineSplitter splitter = new LineSplitter();
+  var lines = splitter.convert(output);
+  // Looks like this:
+  // Total: 15556 tests
+  var total = int.parse(lines[0].split(' ')[1].trim());
+  var count = 0;
+  for (var i = 1; i < lines.length; i++) {
+    if (lines[i] == '') continue;
+    // Looks like this:
+    //  * 3218 tests will be skipped (3047 skipped by design)
+    count += int.parse(lines[i].split(' ')[2].trim());
+  }
+  if (count != total) {
+    print('Count: $count, total: $total');
+    throw 'Foobar';

[ricow1, 2015/01/13 08:03:01]

Trow a better string than Foobar here:
This is just a sanity check, but maybe:
throw 'Counts don't add up to total, please validate manually';

[kevmoo, 2015/01/13 19:27:35]

Done.

+  }
+}
+
+void main(List<String> args) {
+  ensureBuild();
+
+  List<String> keys;
+  for (var combination in getCombinations()) {
+    for (var mode in combination['modes']) {
+      for (var arch in combination['archs']) {
+        for (var runtime in combination['runtimes']) {
+          var compiler = combination['compiler'];
+
+          var args = ['tools/test.py', '-m$mode', '-c$compiler', '-r$runtime',
+                      '-a$arch', '--json-only', '--use-sdk'];
+          var result = Process.runSync('python', args);
+          if (result.exitCode != 0) {
+            print(result.stdout);
+            print(result.stderr);
+            throw "Error running";

[ricow1, 2015/01/13 08:03:00]

'Error running: ${args.join(" ")}';

[kevmoo, 2015/01/13 19:27:35]

Done.

+          }
+
+          // Find Total: 15063 tests
+          // Everything after this will be the report.
+          var totalIndex = result.stdout.indexOf('JSON:');
+          var report = result.stdout.substring(totalIndex+5);

[ricow1, 2015/01/13 08:03:01]

space around +

[kevmoo, 2015/01/13 19:27:35]

Done.

+
+          var map = JSON.decode(report) as Map;
+
+          if (keys == null) {
+            keys = map.keys.toList();
+            var firstKey = keys.removeAt(0);
+            if (firstKey != 'total') {
+              throw '"total" should be the first key';
+            }
+
+            var headers = ['compiler', 'runtime', 'arch', 'mode', 'total'];
+            for (var k in keys) {
+              headers.addAll([k, '${k}_pct']);
+            }
+            print(headers.join(','));
+          }
+
+          var total = map['total'];
+          var values = [compiler, runtime, arch, mode, total];
+
+          for (var key in keys) {
+            var value = map[key];
+            values.add(value);
+            var pct = 100*(value/total);
+            values.add('${pct.toStringAsFixed(3)}%');
+          }
+
+          print(values.join(','));
+        }
+      }
+    }
+  }
+}