sandwich: Make metrics extraction more customizable.

tools/android/loading/sandwich_task_builder.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import csv
 import logging
 import json
+import logging
 import os
 import shutil
 
 import chrome_cache
 import common_util
 import emulation
+from loading_trace import LoadingTrace

[pasko, 2016/05/31 16:00:01]

more consistent: 'import loading_trace'.

[gabadie, 2016/06/01 12:04:34]

Done.

 import sandwich_metrics
 import sandwich_misc
 import sandwich_runner
 import task_manager
 
 
 def NetworkSimulationTransformer(network_condition):
   """Creates a function that accepts a SandwichRunner as a parameter and sets
   network emulation options on it.
 
@@ skipping 155 matching lines @@
           SandwichRunner.Run() in the given order.
 
     Here is the full dependency of the added tree for the returned task:
     <transformer_list_name>/<subresource_discoverer>-metrics.csv
       depends on: <transformer_list_name>/<subresource_discoverer>-run/
         depends on: common/<subresource_discoverer>-cache.zip
           depends on: some tasks saved by PopulateCommonPipelines()
           depends on: common/<subresource_discoverer>-setup.json
             depends on: some tasks saved by PopulateCommonPipelines()
     """
+    ADDITIONAL_COLUMN_NAMES = [
+        'url',
+        'repeat_id',
+        'subresource_discoverer',
+        'subresource_count',
+        # The amount of subresources detected at SetupBenchmark step.
+        'subresource_count_theoretic',
+        # Amount of subresources for caching as suggested by the subresource
+        # discoverer.
+        'cached_subresource_count_theoretic',
+        'cached_subresource_count']
+
     assert subresource_discoverer in sandwich_misc.SUBRESOURCE_DISCOVERERS
     assert 'common' not in sandwich_misc.SUBRESOURCE_DISCOVERERS
     shared_task_prefix = os.path.join('common', subresource_discoverer)
     task_prefix = os.path.join(transformer_list_name, subresource_discoverer)
 
     @self.RegisterTask(shared_task_prefix + '-setup.json', merge=True,
                        dependencies=[self._subresources_for_urls_task])
     def SetupBenchmark():
       whitelisted_urls = sandwich_misc.ExtractDiscoverableUrls(
           self._trace_from_grabbing_reference_cache, subresource_discoverer)
@@ skipping 27 matching lines @@
       runner.wpr_out_log_path = os.path.join(
           RunBenchmark.path, sandwich_runner.WPR_LOG_FILENAME)
       runner.cache_archive_path = BuildBenchmarkCacheArchive.path
       runner.cache_operation = sandwich_runner.CacheOperation.PUSH
       runner.output_dir = RunBenchmark.path
       runner.Run()
 
     @self.RegisterTask(task_prefix + '-metrics.csv',
                        dependencies=[RunBenchmark])
     def ExtractMetrics():
+      # TODO PERF IMPROVMENT(gabadie): Can load trace only once an use it for

[pasko, 2016/05/31 16:00:01]

do you want to make this improvement in a followup change or in this one?

[pasko, 2016/05/31 16:00:01]

The format of TODO is somewhat strict to allow some grepping, so we'd be better
with:
# TODO(gabadie): Performance improvement: load each trace only once and use it
for validation and extraction of metrics later.

[gabadie, 2016/06/01 12:04:34]

Separate.

[gabadie, 2016/06/01 12:04:34]

Done.

+      # validation and metrics extraction.
       sandwich_misc.VerifyBenchmarkOutputDirectory(
           SetupBenchmark.path, RunBenchmark.path)
-      trace_metrics_list = \
-          sandwich_metrics.ExtractMetricsFromRunnerOutputDirectory(
-              SetupBenchmark.path, RunBenchmark.path)
-      trace_metrics_list.sort(key=lambda e: e['repeat_id'])
+
+      benchmark_setup = json.load(open(SetupBenchmark.path))
+      run_metrics_list = []
+      for repeat_id, repeat_dir in sandwich_runner.WalkRepeatedRuns(
+          RunBenchmark.path):
+        trace_path = os.path.join(repeat_dir, sandwich_runner.TRACE_FILENAME)
+        logging.info('processing trace \'%s\'' % trace_path)

[pasko, 2016/05/31 16:00:01]

logging.info('processing trace: %s', trace_path)

[gabadie, 2016/06/01 12:04:34]

Done.

+        trace = LoadingTrace.FromJsonFile(trace_path)
+        run_metrics = {
+            'url': trace.url,
+            'repeat_id': repeat_id,
+            'subresource_discoverer': benchmark_setup['subresource_discoverer'],
+            'subresource_count': len(sandwich_misc.ListUrlRequests(
+                trace, sandwich_misc.RequestOutcome.All)),
+            'subresource_count_theoretic':
+                len(benchmark_setup['url_resources']),
+            'cached_subresource_count': len(sandwich_misc.ListUrlRequests(
+                trace, sandwich_misc.RequestOutcome.ServedFromCache)),
+            'cached_subresource_count_theoretic':
+                len(benchmark_setup['cache_whitelist']),
+        }
+        run_metrics.update(
+            sandwich_metrics.ExtractCommonMetricsFromRepeatDirectory(
+                repeat_dir, trace))
+        run_metrics_list.append(run_metrics)
+
+      run_metrics_list.sort(key=lambda e: e['repeat_id'])
       with open(ExtractMetrics.path, 'w') as csv_file:
-        writer = csv.DictWriter(csv_file,
-                                fieldnames=sandwich_metrics.CSV_FIELD_NAMES)
+        writer = csv.DictWriter(csv_file, fieldnames=(ADDITIONAL_COLUMN_NAMES +
+                                    sandwich_metrics.COMMON_CSV_COLUMN_NAMES))
         writer.writeheader()
-        for trace_metrics in trace_metrics_list:
+        for trace_metrics in run_metrics_list:
           writer.writerow(trace_metrics)
 
     self._common_builder.default_final_tasks.append(ExtractMetrics)