sandwich: Implements filter-cache sub-command.

tools/android/loading/sandwich.py

 #! /usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Instructs Chrome to load series of web pages and reports results.
 
 When running Chrome is sandwiched between preprocessed disk caches and
 WepPageReplay serving all connections.
 
@@ skipping 17 matching lines @@
 from devil.android import device_utils
 
 sys.path.append(os.path.join(_SRC_DIR, 'build', 'android'))
 from pylib import constants
 import devil_chromium
 
 import chrome_cache
 import chrome_setup
 import device_setup
 import devtools_monitor
+import frame_load_lens
+import loading_trace
 import options
 import page_track
 import pull_sandwich_metrics
+import request_dependencies_lens
 import trace_recorder
 import tracing
 import wpr_backend
 
 
 # Use options layer to access constants.
 OPTIONS = options.OPTIONS
 
 _JOB_SEARCH_PATH = 'sandwich_jobs'
 
@@ skipping 145 matching lines @@
   def _RunNavigation(self, url, clear_cache, trace_id=None):
     with device_setup.DeviceConnection(
         device=self._device,
         additional_flags=self._chrome_additional_flags) as connection:
       additional_metadata = {}
       if self._GetEmulatorNetworkCondition('browser'):
         additional_metadata = chrome_setup.SetUpEmulationAndReturnMetadata(
             connection=connection,
             emulated_device_name=None,
             emulated_network_name=self._GetEmulatorNetworkCondition('browser'))
-      loading_trace = trace_recorder.MonitorUrl(
+      trace = trace_recorder.MonitorUrl(
           connection, url,
           clear_cache=clear_cache,
           categories=pull_sandwich_metrics.CATEGORIES,
           timeout=_DEVTOOLS_TIMEOUT)
-      loading_trace.metadata.update(additional_metadata)
+      trace.metadata.update(additional_metadata)
       if trace_id != None and self.trace_output_directory:
-        loading_trace_path = os.path.join(
+        trace_path = os.path.join(
             self.trace_output_directory, str(trace_id), 'trace.json')
-        os.makedirs(os.path.dirname(loading_trace_path))
-        loading_trace.ToJsonFile(loading_trace_path)
+        os.makedirs(os.path.dirname(trace_path))
+        trace.ToJsonFile(trace_path)
 
   def _RunUrl(self, url, trace_id=0):
     clear_cache = False
     if self.cache_operation == 'clear':
       clear_cache = True
     elif self.cache_operation == 'push':
       self._device.KillAll(OPTIONS.chrome_package_name, quiet=True)
       chrome_cache.PushBrowserCache(self._device,
                                     self._local_cache_directory_path)
     elif self.cache_operation == 'reload':
@@ skipping 133 matching lines @@
   create_cache_parser = subparsers.add_parser('pull-metrics',
       help='Pulls metrics in CSV from a run\'s loading trace.')
   create_cache_parser.add_argument('--traces-directory', required=True,
                                    dest='trace_output_directory', type=str,
                                    help='Path of loading traces directory.')
   create_cache_parser.add_argument('--out-metrics', default=None, type=str,
                                    dest='metrics_csv_path',
                                    help='Path where to save the metrics\'s '+
                                       'CSV.')
 
+  # Filter cache subcommand.
+  filter_cache_parser = subparsers.add_parser('filter-cache',
+      help='Cache filtering that keeps only resources discoverable by the HTML'+
+          ' document parser.')
+  filter_cache_parser.add_argument('--cache-archive', type=str, required=True,
+                                   dest='cache_archive_path',
+                                   help='Path of the cache archive to filter.')
+  filter_cache_parser.add_argument('--output', type=str, required=True,
+                                   dest='output_cache_archive_path',
+                                   help='Path of filtered cache archive.')
+  filter_cache_parser.add_argument('loading_trace_paths', type=str, nargs='+',

[pasko, 2016/02/26 17:10:06]

underscores -> dashes, please

[gabadie, 2016/03/01 10:40:48]

Nope. Because non optional argument. I would have added them if this was
--loading-trace-paths meaning you need to write those word in the command line.
But here you won't write it, you just directly list the paths of the loading
traces.

[pasko, 2016/03/01 17:55:53]

ah, I see, thanks, maybe move the definition of this non-optional argument above
all optional ones? That is my implicit assumption, not sure where it is coming
from, maybe protobufs I saw were all like that.

+                                   metavar='LOADING_TRACE',
+                                   help='A loading trace path to generate the' +

[pasko, 2016/02/26 17:10:06]

please document more details on how they are generated? is it everything except
data urls.

I have less confidence in trace putting all urls correctly than the cache tool
reading entries one by one and deleting them if they are not data urls, this
would depend on a smaller code surface, I think.

WDYT?

[gabadie, 2016/03/01 10:40:48]

Done.
You might be mixing up between traces and clovis loading traces.
https://code.google.com/p/chromium/codesearch#chromium/src/tools/android/load....
The point of using a loading traces is to create the resource dependency graph
that clovis guys have implemented to actually white list only resources
discover-able by the html pre-scanner.

[pasko, 2016/03/01 17:55:53]

No, I was not mixing them this time. We can name the latter as "extended traces"
or "clovis traces" btw.

The Traces of Mister Clovis :)

Anyway, I have relatively low level of trust for the metadata devtools interface
provides. It is less mission-critical than, say HTMLPreloadScanner, and there
are known bugs which droger@ has been looking at, right?

I added a task to check the equivalence later.

+                                      ' urls white-list from.')
+
   return parser
 
 
 def _RecordWprMain(args):
   sandwich_runner = SandwichRunner(args.job)
   sandwich_runner.PullConfigFromArgs(args)
   sandwich_runner.wpr_record = True
   sandwich_runner.PrintConfig()
   if not os.path.isdir(os.path.dirname(args.wpr_archive_path)):
     os.makedirs(os.path.dirname(args.wpr_archive_path))
@@ skipping 55 matching lines @@
   trace_metrics_list.sort(key=lambda e: e['id'])
   with open(args.metrics_csv_path, 'w') as csv_file:
     writer = csv.DictWriter(csv_file,
                             fieldnames=pull_sandwich_metrics.CSV_FIELD_NAMES)
     writer.writeheader()
     for trace_metrics in trace_metrics_list:
       writer.writerow(trace_metrics)
   return 0
 
 
+def _FilterCacheMain(args):
+  whitelisted_urls = set()
+  for loading_trace_path in args.loading_trace_paths:
+    logging.info('loading %s' % loading_trace_path)
+    trace = loading_trace.LoadingTrace.FromJsonFile(loading_trace_path)
+    requests_lens = request_dependencies_lens.RequestDependencyLens(trace)
+    deps = requests_lens.GetRequestDependencies()
+
+    main_resource_request = deps[0][0]
+    logging.info('white-listing %s' % main_resource_request.url)
+    whitelisted_urls.add(main_resource_request.url)
+    for (first, second, reason) in deps:
+      # Ignore data protocols.
+      if not second.protocol.startswith('http'):
+        continue
+      if (first.request_id == main_resource_request.request_id and
+          reason == 'parser' and second.url not in whitelisted_urls):
+        logging.info('white-listing %s' % second.url)
+        whitelisted_urls.add(second.url)
+
+  if not os.path.isdir(os.path.dirname(args.output_cache_archive_path)):
+    os.makedirs(os.path.dirname(args.output_cache_archive_path))
+  chrome_cache.ApplyUrlWhitelistToCacheArchive(args.cache_archive_path,
+                                               whitelisted_urls,
+                                               args.output_cache_archive_path)
+  return 0
+
+
 def main(command_line_args):
   logging.basicConfig(level=logging.INFO)
   devil_chromium.Initialize()
 
   # Don't give the argument yet. All we are interested in for now is accessing
   # the default values of OPTIONS.
   OPTIONS.ParseArgs([])
 
   args = _ArgumentParser().parse_args(command_line_args)
 
   if args.subcommand == 'record-wpr':
     return _RecordWprMain(args)
   if args.subcommand == 'patch-wpr':
     return _PatchWprMain(args)
   if args.subcommand == 'create-cache':
     return _CreateCacheMain(args)
   if args.subcommand == 'run':
     return _RunJobMain(args)
   if args.subcommand == 'pull-metrics':
     return _PullMetricsMain(args)
+  if args.subcommand == 'filter-cache':
+    return _FilterCacheMain(args)
   assert False
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))