mojo_benchmark: aggregate results over multiple runs.

mojo/devtools/common/devtoolslib/benchmark.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Logic that drives runs of the benchmarking mojo app and parses its output."""
 
 import os.path
 import re
 
 _BENCHMARK_APP = 'https://core.mojoapps.io/benchmark.mojo'
@@ skipping 25 matching lines @@
     if match:
       measurement_spec = match.group(1)
       measurement_result = match.group(2)
       try:
         measurement_results[measurement_spec] = float(measurement_result)
       except ValueError:
         pass
   return measurement_results
 
 
-class Results(object):
+class Outcome(object):
   """Holds results of a benchmark run."""
 
   def __init__(self, succeeded, error_str, output):
     self.succeeded = succeeded
     self.error_str = error_str
     self.output = output
-    self.measurements = None
+    # Maps measurement specs to measurement results given as floats. Only
+    # measurements that succeeded (ie. we retrieved their results) are
+    # represented.
+    self.results = {}
+    self.some_measurements_failed = False
 
 
 def run(shell, shell_args, app, duration_seconds, measurements, verbose,
         android, output_file):
   """Runs the given benchmark by running `benchmark.mojo` in mojo shell with
   appropriate arguments and returns the produced output.
 
   Returns:
-    A tuple of (succeeded, error_msg, output).
+    An instance of Outcome holding the results of the run.
   """
   timeout = duration_seconds + _EXTRA_TIMEOUT
   benchmark_args = []
   benchmark_args.append('--app=' + app)
   benchmark_args.append('--duration=' + str(duration_seconds))
 
   device_output_file = None
   if output_file:
     if android:
       device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file)
       benchmark_args.append('--trace-output=' + device_output_file)
     else:
       benchmark_args.append('--trace-output=' + output_file)
 
   for measurement in measurements:
     benchmark_args.append(measurement['spec'])
 
   shell_args = list(shell_args)
   shell_args.append(_BENCHMARK_APP)
   shell_args.append('--force-offline-by-default')
   shell_args.append('--args-for=%s %s' % (_BENCHMARK_APP,
                                           ' '.join(benchmark_args)))
 
   if verbose:
     print 'shell arguments: ' + str(shell_args)
   return_code, output, did_time_out = shell.run_and_get_output(
       shell_args, timeout=timeout)
 
   if did_time_out:
-    return Results(False, 'timed out', output)
+    return Outcome(False, 'timed out', output)
   if return_code:
-    return Results(False, 'return code: ' + str(return_code), output)
+    return Outcome(False, 'return code: ' + str(return_code), output)
 
   # Pull the trace file even if some measurements are missing, as it can be
   # useful in debugging.
   if device_output_file:
     shell.pull_file(device_output_file, output_file, remove_original=True)
 
-  results = Results(True, None, output)
-  results.measurements = _parse_measurement_results(output)
-  return results
+  outcome = Outcome(True, None, output)
+  parsed_results = _parse_measurement_results(output)
+  for measurement in measurements:
+    spec = measurement['spec']
+    if spec in parsed_results:
+      outcome.results[spec] = parsed_results[spec]
+    else:
+      outcome.some_measurements_failed = True
+  return outcome