A/B fasta performance measurement harness

pkg/front_end/tool/fasta/compile.dart

Index: pkg/front_end/tool/fasta/compile.dart
diff --git a/pkg/front_end/tool/fasta/compile.dart b/pkg/front_end/tool/fasta/compile.dart
index fc7ccb3ce75226fc5e05cbe4c2ae6725056d6dfe..5deaaaddb22f1e64122ae989d3e20903fa6260ac 100644
--- a/pkg/front_end/tool/fasta/compile.dart
+++ b/pkg/front_end/tool/fasta/compile.dart
@@ -2,6 +2,48 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'package:front_end/src/fasta/outline.dart' show compileEntryPoint;
+import 'dart:math';
 
-main(List<String> arguments) => compileEntryPoint(arguments);
+import 'package:front_end/src/fasta/outline.dart' as outline;
+
+import 'standard_deviation.dart';
+
+const int iterations = const int.fromEnvironment("iterations", defaultValue: 1);
+
+main(List<String> arguments) async {
+  // Timing results for each iteration
+  List<double> elapseTimes = <double>[];
+
+  for (int i = 0; i < iterations; i++) {
+    if (i > 0) {
+      print("\n");
+    }
+
+    var stopwatch = new Stopwatch()..start();
+    await outline.compile(arguments);
+    stopwatch.stop();
+
+    elapseTimes.add(stopwatch.elapsedMilliseconds.toDouble());
+  }
+
+  // No summary if less than 4 iterations
+  if (elapseTimes.length < 4) {

[Paul Berry, 2017/03/06 20:27:40]

This should be 5.  We need to ensure that at least two values are sent to the
standardDeviation() function, or it will divide by zero.

[danrubel, 2017/03/09 02:01:01]

Good point. Addressed in https://codereview.chromium.org/2739963002

+    return;
+  }
+
+  // Calculate the mean of warm runs (#4 to n)
+  List<double> warmTimes = elapseTimes.sublist(3);
+  double mean = average(warmTimes);
+
+  // Calculate the standard deviation
+  double stdDev = standardDeviation(mean, warmTimes);
+
+  // Calculate the standard deviation of the mean
+  double stdDevOfTheMean = standardDeviationOfTheMean(warmTimes, stdDev);
+
+  print('Summary:');
+  print('  Elapse times: $elapseTimes');
+  print('  Cold start (first run): ${elapseTimes[0]}');
+  print('  Warm run average (runs #4 to #$iterations): $mean');
+  print('  Warm run standard deviation of the mean: $stdDevOfTheMean');
+}