Final touches for running smoketests.

tools/testing/perf_testing/smoketest/BenchmarkBase.dart

+// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/**
+ * The superclass from which all benchmarks inherit from.

[Siggi Cherem (dart-lang), 2011/12/15 19:37:33]

nit: we tend to do the 1-line style doc (/** comment */) when the comment fits
in one line.
(here and below)

[Emily Fortuna, 2011/12/15 23:16:10]

Done.

+ */
+class BenchmarkBase {
+  /* Benchmark name. */

[Siggi Cherem (dart-lang), 2011/12/15 19:37:33]

nit: -> add one more * (/**) 
(here and below)

[Emily Fortuna, 2011/12/15 23:16:10]

Done.

+  final String name;
+
+  const BenchmarkBase(String name) : this.name = name;
+
+  /**
+   * The benchmark code.
+   * This function is not used, if both [warmup] and [exercise] are overwritten.
+   */
+  void run() { }
+
+  /**
+   * Runs a short version of the benchmark. By default invokes [run] once.
+   */
+  void warmup() {
+    run();
+  }
+
+  /**
+   * Exercices the benchmark. By default invokes [run] 10 times.
+   */
+  void exercise() {
+    for (int i = 0; i < 10; i++) {
+      run();
+    }
+  }
+
+  /**
+   * Not measured setup code executed prior to the benchmark runs.
+   */
+  void setup() { }
+
+  /**
+   * Not measures teardown code executed after the benchark runs.
+   */
+  void teardown() { }
+
+  /**
+   * Measures the score for this benchmark by executing it repeately until
+   * time minimum has been reached.
+   */
+  static double measureFor(Function f, int timeMinimum) {
+    int time = 0;
+    int iter = 0;
+    Stopwatch watch = new Stopwatch();
+    watch.start();
+    int elapsed = 0;
+    while (elapsed < timeMinimum || iter < 32) {
+      f();
+      elapsed = watch.elapsedInMs();
+      iter++;
+    }
+    return (1000.0 * iter) / elapsed;
+  }
+
+  /**
+   * Measures the score for the benchmark and returns it.
+   * We measure iterations / sec (so bigger = better!).
+   */
+  double measure() {
+    setup();
+    // Warmup for at least 1000ms. Discard result.
+    measureFor(() { this.warmup(); }, 1000);
+    // Run the benchmark for at least 1000ms.
+    double result = measureFor(() { this.exercise(); }, 1000);
+    teardown();
+    return result;
+  }
+
+  void report() {
+    num score = measure();
+    Map<String, int> normalizingDict = {'Smoketest': 100};
+    window.console.log(name + " " + score.toString());
+    score = score / normalizingDict[name];
+    BenchmarkSuite.ONLY.updateIndividualScore(name, score);
+  }
+}
+
+/** 
+ * The controller class that runs all of the benchmarks.
+ */
+class BenchmarkSuite {
+  /* The set of benchmarks that have yet to run. */
+  List<Function> benchmarks;
+  /**

[Siggi Cherem (dart-lang), 2011/12/15 19:37:33]

nit: + empty line

[Emily Fortuna, 2011/12/15 23:16:10]

Done.

+   * The set of scores from the benchmarks that have already run. (Used for
+   * calculating the Geometric mean).
+   */
+  List<num> scores;
+
+  /* The total number of benchmarks we will be running. */
+  int totalBenchmarks;
+
+  /* Singleton pattern: There's only one BenchmarkSuite. */
+  static BenchmarkSuite _ONLY = null;
+
+  BenchmarkSuite._internal() {
+    scores = [];
+    benchmarks = [() => Smoketest.main()];
+    totalBenchmarks = benchmarks.length;
+  }
+
+  /* Accessor for our Singleton variable. */
+  static BenchmarkSuite get ONLY() {

[Siggi Cherem (dart-lang), 2011/12/15 19:37:33]

now that we have top-level getters, this would look nicer at the top level of
the library. for instance:

BenchmarkSuite get BENCHMARK_SUITE() {
 ...
}

[Emily Fortuna, 2011/12/15 23:16:10]

Done.

+    if (_ONLY == null) {
+      _ONLY = new BenchmarkSuite._internal();
+    }
+    return _ONLY;
+  }
+  
+  /* Run all of the benchmarks that we have in our benchmarks list. */
+  runBenchmarks() {
+    runBenchmarksHelper(benchmarks);
+  }
+ 
+  /**
+   * Run the remaining benchmarks in our list. We chain the calls providing
+   * little breaks for the main page to gain control, so we don't force the 
+   * entire page to hang the whole time.
+   */
+  runBenchmarksHelper(benchmarks) {

[Siggi Cherem (dart-lang), 2011/12/15 19:37:33]

here a type for the argument would be helpful (e.g. List<Function>)

do you need to pass this argument? (benchmarks is also a field in this class)

If you do, can this be a static method? can you name the argument differently so
that it doesn't shadow the field name?

[Emily Fortuna, 2011/12/15 23:16:10]

I need to pass this argument so that there are brief breaks of control between
each benchmark. See comment on line 134. 

Other comments: fixed.

On 2011/12/15 19:37:33, sigmund wrote:

+    // Remove the last benchmark, and run it.
+    var benchmark = benchmarks.removeLast();
+    benchmark();
+    if (benchmarks.length > 0) {
+      /* Provide small breaks between each benchmark, so that the browser 
+      doesn't get unhappy about long running scripts, and so the user
+      can regain control of the UI to kill the page as needed. */
+      window.setTimeout(() => runBenchmarksHelper(benchmarks), 25);
+    } else if (benchmarks.length == 0) {
+      // We've run all of the benchmarks. Update the page with the score.
+      BenchmarkView.ONLY.setScore(geometricMean(scores));
+    }
+  }
+
+  /* Store the results of a single benchmark run. */
+  updateIndividualScore(String name, num score) {
+    scores.add(score); 
+    BenchmarkView.ONLY.incrementProgress(name, score, totalBenchmarks);
+  }
+
+  /* Computes the geometric mean of a set of numbers. */
+  geometricMean(numbers) {
+    num log = 0;
+    for (num n in numbers) {
+      log += Math.log(n);
+    }
+    return Math.pow(Math.E, log / numbers.length);
+  }
+}
+
+/* Controls how results are displayed to the user, by updating the HTML. */
+class BenchmarkView {
+
+  /* The number of benchmarks that have finished executing. */
+  int numCompleted = 0;
+
+  /* Singleton pattern: There's only one BenchmarkSuite. */
+  static BenchmarkView _ONLY = null;
+
+  BenchmarkView._internal();
+
+  /* Accessor for our Singleton variable. */
+  static BenchmarkView get ONLY() {

[Siggi Cherem (dart-lang), 2011/12/15 19:37:33]

same here (move to top-level)

[Emily Fortuna, 2011/12/15 23:16:10]

Done.

+    if (_ONLY == null) {
+      _ONLY = new BenchmarkView._internal();
+    }
+    return _ONLY;
+  }
+
+  /* Update the page HTML to show the calculated score. */
+  setScore(num score) {
+    String newScore = formatScore(score * 100.0);
+    Element status = document.query("#status");
+    status.innerHTML =  "Score: $newScore <br>";
+  }
+ 
+  /**
+   * Update the page HTML to show how much progress we've made through the 
+   * benchmarks.
+   */
+  incrementProgress(String name, num score, num totalBenchmarks) {
+    String newScore = formatScore(score * 100.0);
+    Element results = document.query("#results");
+    results.innerHTML +=  "$name: $newScore <br>";
+
+    Element status = document.query("#status");
+    numCompleted++;
+    // Slightly incorrect (truncating) percentage, but this is just to show
+    // the user we're making progress.
+    num percentage = 100 * numCompleted ~/ totalBenchmarks;
+    status.innerHTML = "Running: $percentage% completed.";
+  }
+  
+  /** 
+   * Rounds the score to have at least three significant digits (hopefully)
+   * helping readability of the scores.
+   */
+  String formatScore(num value) {
+    if (value > 100) {
+      return value.toStringAsFixed(0);
+    } else {
+      return value.toStringAsFixed(2);
+    }
+  }
+}