sandwich: Implements filter-cache sub-command.

tools/android/loading/sandwich.py

 #! /usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Instructs Chrome to load series of web pages and reports results.
 
 When running Chrome is sandwiched between preprocessed disk caches and
 WepPageReplay serving all connections.
 
@@ skipping 17 matching lines @@
 from devil.android import device_utils
 
 sys.path.append(os.path.join(_SRC_DIR, 'build', 'android'))
 from pylib import constants
 import devil_chromium
 
 import chrome_cache
 import chrome_setup
 import device_setup
 import devtools_monitor
+import frame_load_lens
+import loading_trace
 import options
 import page_track
 import pull_sandwich_metrics
+import request_dependencies_lens
 import trace_recorder
 import tracing
 import wpr_backend
 
 
 # Use options layer to access constants.
 OPTIONS = options.OPTIONS
 
 _JOB_SEARCH_PATH = 'sandwich_jobs'
 
@@ skipping 148 matching lines @@
   def _RunNavigation(self, url, clear_cache, trace_id=None):
     with device_setup.DeviceConnection(
         device=self._device,
         additional_flags=self._chrome_additional_flags) as connection:
       additional_metadata = {}
       if self._GetEmulatorNetworkCondition('browser'):
         additional_metadata = chrome_setup.SetUpEmulationAndReturnMetadata(
             connection=connection,
             emulated_device_name=None,
             emulated_network_name=self._GetEmulatorNetworkCondition('browser'))
-      loading_trace = trace_recorder.MonitorUrl(
+      trace = trace_recorder.MonitorUrl(
           connection, url,
           clear_cache=clear_cache,
           categories=pull_sandwich_metrics.CATEGORIES,
           timeout=_DEVTOOLS_TIMEOUT)
-      loading_trace.metadata.update(additional_metadata)
+      trace.metadata.update(additional_metadata)
       if trace_id != None and self.trace_output_directory:
-        loading_trace_path = os.path.join(
+        trace_path = os.path.join(
             self.trace_output_directory, str(trace_id), 'trace.json')
-        os.makedirs(os.path.dirname(loading_trace_path))
-        loading_trace.ToJsonFile(loading_trace_path)
+        os.makedirs(os.path.dirname(trace_path))
+        trace.ToJsonFile(trace_path)
 
   def _RunUrl(self, url, trace_id=0):
     clear_cache = False
     if self.cache_operation == 'clear':
       clear_cache = True
     elif self.cache_operation == 'push':
       self._device.KillAll(OPTIONS.chrome_package_name, quiet=True)
       chrome_cache.PushBrowserCache(self._device,
                                     self._local_cache_directory_path)
     elif self.cache_operation == 'reload':
@@ skipping 132 matching lines @@
   create_cache_parser = subparsers.add_parser('extract-metrics',
       help='Extracts metrics from a loading trace and saves as CSV.')
   create_cache_parser.add_argument('--trace-directory', required=True,
                                    dest='trace_output_directory', type=str,
                                    help='Path of loading traces directory.')
   create_cache_parser.add_argument('--out-metrics', default=None, type=str,
                                    dest='metrics_csv_path',
                                    help='Path where to save the metrics\'s '+
                                       'CSV.')
 
+  # Filter cache subcommand.
+  filter_cache_parser = subparsers.add_parser('filter-cache',
+      help='Cache filtering that keeps only resources discoverable by the HTML'+
+          ' document parser.')
+  filter_cache_parser.add_argument('--cache-archive', type=str, required=True,
+                                   dest='cache_archive_path',
+                                   help='Path of the cache archive to filter.')
+  filter_cache_parser.add_argument('--output', type=str, required=True,
+                                   dest='output_cache_archive_path',
+                                   help='Path of filtered cache archive.')
+  filter_cache_parser.add_argument('loading_trace_paths', type=str, nargs='+',

[pasko, 2016/03/01 16:01:11]

nit: loading-trace-paths for consistency

[gabadie, 2016/03/01 16:40:00]

This change would be point less because this is a positional argument. So the
only place it is shown is in the man, with underscore, and it won't be in
command line. And since I have not explicitly overridden the dest parameter, I
would rather to keep here with the underscore for easier grep of the code for
this variable name.

[pasko, 2016/03/01 17:55:53]

Yes, we discussed it in another review. Good to go.

+      metavar='LOADING_TRACE',
+      help='A loading trace path generated by a sandwich run for a given url.' +

[pasko, 2016/03/01 16:01:11]

I would say: "A list of extended traces generated by .."

To avoid overloading the term "trace", shall we name the thing "extended trace"?

Then I would actually name the argument as: --extended-traces. WDYT?

[gabadie, 2016/03/01 16:40:00]

Done.
I agree the loading trace is terrible. But I would rather stick with the name of
the python class (LoadingTrace) that can load this LoadingTrace.

+          ' This is used to have a resource dependency graph to white-list ' +
+          'the ones discoverable by the HTML pre-scanner for that given url.')
+
   return parser
 
 
 def _RecordWprMain(args):
   sandwich_runner = SandwichRunner(args.job)
   sandwich_runner.PullConfigFromArgs(args)
   sandwich_runner.wpr_record = True
   sandwich_runner.PrintConfig()
   if not os.path.isdir(os.path.dirname(args.wpr_archive_path)):
     os.makedirs(os.path.dirname(args.wpr_archive_path))
@@ skipping 56 matching lines @@
   trace_metrics_list.sort(key=lambda e: e['id'])
   with open(args.metrics_csv_path, 'w') as csv_file:
     writer = csv.DictWriter(csv_file,
                             fieldnames=pull_sandwich_metrics.CSV_FIELD_NAMES)
     writer.writeheader()
     for trace_metrics in trace_metrics_list:
       writer.writerow(trace_metrics)
   return 0
 
 
+def _FilterCacheMain(args):
+  whitelisted_urls = set()
+  for loading_trace_path in args.loading_trace_paths:
+    logging.info('loading %s' % loading_trace_path)
+    trace = loading_trace.LoadingTrace.FromJsonFile(loading_trace_path)
+    requests_lens = request_dependencies_lens.RequestDependencyLens(trace)
+    deps = requests_lens.GetRequestDependencies()
+
+    main_resource_request = deps[0][0]
+    logging.info('white-listing %s' % main_resource_request.url)
+    whitelisted_urls.add(main_resource_request.url)
+    for (first, second, reason) in deps:
+      # Ignore data protocols.
+      if not second.protocol.startswith('http'):
+        continue
+      if (first.request_id == main_resource_request.request_id and
+          reason == 'parser' and second.url not in whitelisted_urls):
+        logging.info('white-listing %s' % second.url)
+        whitelisted_urls.add(second.url)
+
+  if not os.path.isdir(os.path.dirname(args.output_cache_archive_path)):
+    os.makedirs(os.path.dirname(args.output_cache_archive_path))
+  chrome_cache.ApplyUrlWhitelistToCacheArchive(args.cache_archive_path,
+                                               whitelisted_urls,
+                                               args.output_cache_archive_path)
+  return 0
+
+
 def main(command_line_args):
   logging.basicConfig(level=logging.INFO)
   devil_chromium.Initialize()
 
   # Don't give the argument yet. All we are interested in for now is accessing
   # the default values of OPTIONS.
   OPTIONS.ParseArgs([])
 
   args = _ArgumentParser().parse_args(command_line_args)
 
   if args.subcommand == 'record-wpr':
     return _RecordWprMain(args)
   if args.subcommand == 'patch-wpr':
     return _PatchWprMain(args)
   if args.subcommand == 'create-cache':
     return _CreateCacheMain(args)
   if args.subcommand == 'run':
     return _RunJobMain(args)
   if args.subcommand == 'extract-metrics':
     return _ExtractMetricsMain(args)
+  if args.subcommand == 'filter-cache':
+    return _FilterCacheMain(args)
   assert False
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))