cleanup and improve A/B fasta test harness

pkg/front_end/tool/fasta/abcompile.dart

+// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
 import 'dart:async';
 import 'dart:convert';
 import 'dart:math';
 import 'dart:io';
 
 import 'standard_deviation.dart';
 
 const String bRootPath = const String.fromEnvironment("bRoot");
-const int iterations = const int.fromEnvironment("iterations", defaultValue: 1);
+const int abIterations =
+    const int.fromEnvironment("abIterations", defaultValue: 15);
+const int iterations =
+    const int.fromEnvironment("iterations", defaultValue: 15);
 
 /// Compare the performance of two different fast implementations
 /// by alternately launching the compile application in this directory
 /// and the compile application location in the repo specified by "bRoot"
 /// via -DbRoot=/absolute/path/to/other/sdk/repo
 main(List<String> args) async {
   print(args);
   if (bRootPath == null) {
     print('Expected -DbRoot=/absolute/path/to/other/sdk/repo');
     exit(1);
@@ skipping 20 matching lines @@
 
   print('Comparing:');
   print('A: $aCompile');
   print('B: $bCompile');
   print('');
 
   List<double> aCold = <double>[];
   List<double> aWarm = <double>[];
   List<double> bCold = <double>[];
   List<double> bWarm = <double>[];
-  for (int count = 0; count < 15; ++count) {
+
+  var stopwatch = new Stopwatch()..start();
+  for (int count = 0; count < abIterations; ++count) {
+    print('A/B iteration ${count + 1} of $abIterations ...');
     await run(aRoot, aCompile, args, aCold, aWarm);
     await run(bRoot, bCompile, args, bCold, bWarm);
   }
+  stopwatch.stop();
+  print('Overall run time: ${stopwatch.elapsed.inMinutes} minutes');
 
   print('');
   print('Raw data:');
   print('A cold, A warm, B cold, B warm');
   for (int index = 0; index < aCold.length; ++index) {
     print('${aCold[index]}, ${aWarm[index]}, ${bCold[index]}, ${bWarm[index]}');
   }
 
+  if (aWarm.length < 1) {
+    return;
+  }
+
   double aColdMean = average(aCold);
   double aWarmMean = average(aWarm);
   double bColdMean = average(bCold);
   double bWarmMean = average(bWarm);
 
   print('');
   print('Average:');
   print('$aColdMean, $aWarmMean, $bColdMean, $bWarmMean');
 
+  if (aWarm.length < 2) {
+    return;
+  }
+
   double aColdStdDev = standardDeviation(aColdMean, aCold);
   double aWarmStdDev = standardDeviation(aWarmMean, aWarm);
   double bColdStdDev = standardDeviation(bColdMean, bCold);
   double bWarmStdDev = standardDeviation(bWarmMean, bWarm);
 
-  double aColdStdDevMean = standardDeviationOfTheMean(aCold, aColdStdDev);
-  double aWarmStdDevMean = standardDeviationOfTheMean(aWarm, aWarmStdDev);
-  double bColdStdDevMean = standardDeviationOfTheMean(bCold, bColdStdDev);
-  double bWarmStdDevMean = standardDeviationOfTheMean(bWarm, bWarmStdDev);
+  double aColdSDM = standardDeviationOfTheMean(aCold, aColdStdDev);
+  double aWarmSDM = standardDeviationOfTheMean(aWarm, aWarmStdDev);
+  double bColdSDM = standardDeviationOfTheMean(bCold, bColdStdDev);
+  double bWarmSDM = standardDeviationOfTheMean(bWarm, bWarmStdDev);
 
   print('');
   print('Uncertainty:');
-  print(
-      '$aColdStdDevMean, $aWarmStdDevMean, $bColdStdDevMean, $bWarmStdDevMean');
+  print('$aColdSDM, $aWarmSDM, $bColdSDM, $bWarmSDM');
 
   double coldDelta = aColdMean - bColdMean;
-  double coldStdDevMean =
-      sqrt(pow(aColdStdDevMean, 2) + pow(bColdStdDevMean, 2));
+  double coldStdDevMean = sqrt(pow(aColdSDM, 2) + pow(bColdSDM, 2));

[Paul Berry, 2017/03/09 16:58:26]

Nit: this variable should be called "coldUncertainty".  It's confusing to all it
"StdDevMean" because "standard deviation of the mean" is one of several
techniques for computing uncertainty, and it's not even the technique we're
using here (the technique we're using here is "uncertainty propagation").

Similar comment applies to warmStdDevMean, coldSDMPercent, and warmSDMPercent
below.

[danrubel, 2017/03/09 19:05:16]

Good point. Fixed.

   double warmDelta = aWarmMean - bWarmMean;
-  double warmStdDevMean =
-      sqrt(pow(aWarmStdDevMean, 2) + pow(bWarmStdDevMean, 2));
+  double warmStdDevMean = sqrt(pow(aWarmSDM, 2) + pow(bWarmSDM, 2));
+
+  double coldDeltaPercent = (coldDelta / bColdMean * 1000).round() / 10;
+  double coldSDMPercent = (coldStdDevMean / bColdMean * 1000).round() / 10;
+  double warmDeltaPercent = (warmDelta / bWarmMean * 1000).round() / 10;
+  double warmSDMPercent = (warmStdDevMean / bWarmMean * 1000).round() / 10;
+
+  double coldBest = coldDelta - 3 * coldStdDevMean;
+  double coldBestPercent = coldDeltaPercent - 3 * coldSDMPercent;
+  double coldWorst = coldDelta + 3 * coldStdDevMean;
+  double coldWorstPercent = coldDeltaPercent + 3 * coldSDMPercent;
+
+  double warmBest = warmDelta - 3 * warmStdDevMean;
+  double warmBestPercent = warmDeltaPercent - 3 * warmSDMPercent;
+  double warmWorst = warmDelta + 3 * warmStdDevMean;
+  double warmWorstPercent = warmDeltaPercent + 3 * warmSDMPercent;
 
   print('');
   print('Summary:');
-  print('  A cold start - B cold start : $coldDelta');
-  print('  Uncertainty                 : $coldStdDevMean');
+  print('$coldDelta, $coldDeltaPercent%, A cold start - B cold start');
+  print('$coldStdDevMean, $coldSDMPercent%, Propagated uncertainty');
+  print('$coldBest, $coldBestPercent%, 99.9% best case');
+  print('$coldWorst, $coldWorstPercent%, 99.9% worst case');
   print('');
-  print('  A warm runs - B warm runs   : $warmDelta');
-  print('  Uncertainty                 : $warmStdDevMean');
+  print('$warmDelta, $warmDeltaPercent%, A warm runs - B warm runs');
+  print('$warmStdDevMean, $warmSDMPercent%, Propagated uncertainty');
+  print('$warmBest, $warmBestPercent%, 99.9% best case');
+  print('$warmWorst, $warmWorstPercent%, 99.9% worst case');
 }
 
-const String _wroteProgram = 'Wrote program to';
-const String _coldStart = 'Cold start (first run):';
-const String _warmRun = 'Warm run average (runs #4';
+const String _iterationTag = '=== Iteration ';
+const String _summaryTag = 'Summary: {"';
 
 /// Launch the specified dart program, forwarding all arguments and environment
 /// that was passed to this program
 Future<Null> run(Uri workingDir, Uri dartApp, List<String> args,
     List<double> cold, List<double> warm) async {
   print('Running $dartApp');
 
   void processLine(String line) {
-    if (line.contains(_wroteProgram)) {
+    if (line.startsWith(_iterationTag)) {
       // Show progress
       stdout
         ..write('.')
         ..flush();
       return;
     }
-    int index = line.indexOf(_coldStart);
-    if (index >= 0) {
-      cold.add(double.parse(line.substring(index + _coldStart.length)));
-      print('\ncold: ${cold.last}');
-      return;
-    }
-    index = line.indexOf(_warmRun);
-    if (index >= 0) {
-      index = line.indexOf(':', index + _warmRun.length);
-      warm.add(double.parse(line.substring(index + 1)));
-      print('warm: ${warm.last}');
+    if (line.startsWith(_summaryTag)) {
+      String json = line.substring(_summaryTag.length - 2);
+      Map<String, dynamic> results = JSON.decode(json);
+      List<double> elapseTimes = results['elapseTimes'];

[Paul Berry, 2017/03/09 16:58:25]

This should be "elapsedTimes" as well IMHO (here and in lines 159, 160, 161,
163, and 165).

[danrubel, 2017/03/09 19:05:16]

Done.

+      print('\nElapse times: $elapseTimes');
+      if (elapseTimes.length > 0) {
+        cold.add(elapseTimes[0]);
+      }
+      if (elapseTimes.length > 4) {
+        // Drop the first 3 and average the remaining
+        warm.add(average(elapseTimes.sublist(3)));
+      }
       return;
     }
   }
 
   String workingDirPath = workingDir.toFilePath();
   List<String> procArgs = <String>[
     '-Diterations=$iterations',
+    '-Dsummary=true',
     dartApp.toFilePath()
   ];
   procArgs.addAll(args);
 
   Process process = await Process.start(Platform.executable, procArgs,
       workingDirectory: workingDirPath);
   stderr.addStream(process.stderr);
   StreamSubscription<String> stdOutSubscription;
   stdOutSubscription = process.stdout
       .transform(UTF8.decoder)
       .transform(new LineSplitter())
       .listen(processLine, onDone: () {
     stdOutSubscription.cancel();
   }, onError: (e) {
     print('Error: $e');
     stdOutSubscription.cancel();
   });
   int code = await process.exitCode;
   if (code != 0) {
     throw 'fail: $code';
   }
   print('');
 }