sandwich: Make metrics extraction more customizable.

tools/android/loading/sandwich_metrics.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Pull a sandwich run's output directory's metrics from traces into a CSV.
 
 python pull_sandwich_metrics.py -h
 """
 
 import collections
@@ skipping 14 matching lines @@
 from telemetry.internal.image_processing import video
 from telemetry.util import image_util
 from telemetry.util import rgba_color
 
 import loading_trace as loading_trace_module
 import sandwich_runner
 import sandwich_misc
 import tracing
 
 
-CSV_FIELD_NAMES = [
-    'repeat_id',
-    'url',
+COMMON_CSV_COLUMN_NAMES = [
     'chromium_commit',
     'platform',
-    'subresource_discoverer',
-    'subresource_count',
-    # The amount of subresources detected at SetupBenchmark step.
-    'subresource_count_theoretic',
-    # Amount of subresources for caching as suggested by the subresource
-    # discoverer.
-    'cached_subresource_count_theoretic',
-    'cached_subresource_count',
     'first_layout',
     'first_contentful_paint',
     'total_load',
     'js_onload_event',
     'browser_malloc_avg',
     'browser_malloc_max',
     'speed_index',
     'net_emul.name', # Should be in emulation.NETWORK_CONDITIONS.keys()
     'net_emul.download',
     'net_emul.upload',
@@ skipping 144 matching lines @@
     assert attr['units'] == 'bytes'
     size = int(attr['value'], 16)
     browser_malloc_sum += size
     browser_malloc_max = max(browser_malloc_max, size)
   return {
     'browser_malloc_avg': browser_malloc_sum / float(len(browser_dump_events)),
     'browser_malloc_max': browser_malloc_max
   }
 
 
-def _ExtractBenchmarkStatistics(benchmark_setup, loading_trace):
-  """Extracts some useful statistics from a benchmark run.
-
-  Args:
-    benchmark_setup: benchmark_setup: dict representing the benchmark setup
-        JSON. The JSON format is according to:
-            PrefetchBenchmarkBuilder.PopulateLoadBenchmark.SetupBenchmark.
-    loading_trace: loading_trace_module.LoadingTrace.
-
-  Returns:
-    Dictionary with all extracted fields set.
-  """
-  return {
-    'subresource_discoverer': benchmark_setup['subresource_discoverer'],
-    'subresource_count': len(sandwich_misc.ListUrlRequests(
-        loading_trace, sandwich_misc.RequestOutcome.All)),
-    'subresource_count_theoretic': len(benchmark_setup['url_resources']),
-    'cached_subresource_count': len(sandwich_misc.ListUrlRequests(
-        loading_trace, sandwich_misc.RequestOutcome.ServedFromCache)),
-    'cached_subresource_count_theoretic':
-        len(benchmark_setup['cache_whitelist']),
-  }
-
-
 def _ExtractCompletenessRecordFromVideo(video_path):
   """Extracts the completeness record from a video.
 
   The video must start with a filled rectangle of orange (RGB: 222, 100, 13), to
   give the view-port size/location from where to compute the completeness.
 
   Args:
     video_path: Path of the video to extract the completeness list from.
 
   Returns:
@@ skipping 17 matching lines @@
     if total_distance == 0:
       if histogram.Distance(final_histogram) == 0:
         return 1.0
       else:
         return 0.0
     return 1 - histogram.Distance(final_histogram) / total_distance
 
   return [(time, FrameProgress(hist)) for time, hist in histograms]
 
 
-def ComputeSpeedIndex(completeness_record):
+def _ComputeSpeedIndex(completeness_record):
   """Computes the speed-index from a completeness record.
 
   Args:
     completeness_record: list(CompletenessPoint)
 
   Returns:
     Speed-index value.
   """
   speed_index = 0.0
   last_time = completeness_record[0][0]
   last_completness = completeness_record[0][1]
   for time, completeness in completeness_record:
     if time < last_time:
       raise ValueError('Completeness record must be sorted by timestamps.')
     elapsed = time - last_time
     speed_index += elapsed * (1.0 - last_completness)
     last_time = time
     last_completness = completeness
   return speed_index
 
 
-def _ExtractMetricsFromRunDirectory(benchmark_setup, run_directory_path):
-  """Extracts all the metrics from traces and video of a sandwich run.
+def ExtractCommonMetricsFromRepeatDirectory(repeat_dir, trace):
+  """Extracts all the metrics from traces and video of a sandwich run repeat
+  directory.
 
   Args:
-    benchmark_setup: benchmark_setup: dict representing the benchmark setup
-        JSON. The JSON format is according to:
-            PrefetchBenchmarkBuilder.PopulateLoadBenchmark.SetupBenchmark.
-    run_directory_path: Path of the run directory.
+    repeat_dir: Path of the repeat directory within a run directory.
+    trace: preloaded LoadingTrace in |repeat_dir|
+
+  Contract:
+    trace == LoadingTrace.FromJsonFile(
+        os.path.join(repeat_dir, sandwich_runner.TRACE_FILENAME))
 
   Returns:
     Dictionary of extracted metrics.
   """
-  trace_path = os.path.join(run_directory_path, 'trace.json')
-  logging.info('processing trace \'%s\'' % trace_path)
-  loading_trace = loading_trace_module.LoadingTrace.FromJsonFile(trace_path)
   run_metrics = {
-      'url': loading_trace.url,
-      'chromium_commit': loading_trace.metadata['chromium_commit'],
-      'platform': (loading_trace.metadata['platform']['os'] + '-' +
-          loading_trace.metadata['platform']['product_model'])
+      'chromium_commit': trace.metadata['chromium_commit'],
+      'platform': (trace.metadata['platform']['os'] + '-' +
+          trace.metadata['platform']['product_model'])
   }
-  run_metrics.update(_ExtractDefaultMetrics(loading_trace))
-  run_metrics.update(_ExtractMemoryMetrics(loading_trace))
-  if benchmark_setup:
-    run_metrics.update(
-        _ExtractBenchmarkStatistics(benchmark_setup, loading_trace))
-  video_path = os.path.join(run_directory_path, 'video.mp4')
+  run_metrics.update(_ExtractDefaultMetrics(trace))
+  run_metrics.update(_ExtractMemoryMetrics(trace))
+  video_path = os.path.join(repeat_dir, sandwich_runner.VIDEO_FILENAME)
   if os.path.isfile(video_path):
     logging.info('processing speed-index video \'%s\'' % video_path)
     try:
       completeness_record = _ExtractCompletenessRecordFromVideo(video_path)
-      run_metrics['speed_index'] = ComputeSpeedIndex(completeness_record)
+      run_metrics['speed_index'] = _ComputeSpeedIndex(completeness_record)
     except video.BoundingBoxNotFoundException:
       # Sometimes the bounding box for the web content area is not present. Skip
       # calculating Speed Index.
       run_metrics['speed_index'] = _FAILED_CSV_VALUE
   else:
     run_metrics['speed_index'] = _UNAVAILABLE_CSV_VALUE
-  for key, value in loading_trace.metadata['network_emulation'].iteritems():
+  for key, value in trace.metadata['network_emulation'].iteritems():
     run_metrics['net_emul.' + key] = value
+  assert set(run_metrics.keys()) == set(COMMON_CSV_COLUMN_NAMES)
   return run_metrics
-
-
-def ExtractMetricsFromRunnerOutputDirectory(benchmark_setup_path,
-                                            output_directory_path):
-  """Extracts all the metrics from all the traces of a sandwich runner output
-  directory.
-
-  Args:
-    benchmark_setup_path: Path of the JSON of the benchmark setup.
-    output_directory_path: The sandwich runner's output directory to extract the
-        metrics from.
-
-  Returns:
-    List of dictionaries.
-  """
-  benchmark_setup = None
-  if benchmark_setup_path:
-    benchmark_setup = json.load(open(benchmark_setup_path))
-  assert os.path.isdir(output_directory_path)
-  metrics = []
-  for node_name in os.listdir(output_directory_path):
-    if not os.path.isdir(os.path.join(output_directory_path, node_name)):
-      continue
-    try:
-      repeat_id = int(node_name)
-    except ValueError:
-      continue
-    run_directory_path = os.path.join(output_directory_path, node_name)
-    run_metrics = _ExtractMetricsFromRunDirectory(
-        benchmark_setup, run_directory_path)
-    run_metrics['repeat_id'] = repeat_id
-    # TODO(gabadie): Make common metrics extraction with benchmark type
-    # specific CSV column.
-    # assert set(run_metrics.keys()) == set(CSV_FIELD_NAMES)
-    metrics.append(run_metrics)
-  assert len(metrics) > 0, ('Looks like \'{}\' was not a sandwich runner ' +
-                            'output directory.').format(output_directory_path)
-  return metrics