sandwich: Implement SandwichTaskBuilder

tools/android/loading/sandwich_misc.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
+import json
+import os
 
+import chrome_cache
+import common_util
 from loading_trace import LoadingTrace
 from prefetch_view import PrefetchSimulationView
 from request_dependencies_lens import RequestDependencyLens
-from user_satisfied_lens import FirstContentfulPaintLens
 import wpr_backend
 
 
+# Don't prefetch anything

[pasko, 2016/04/14 12:34:42]

# Do not prefetch anything.

(i.e. a full stop at the end)

[gabadie, 2016/04/14 15:43:32]

Done.

+DISABLED_DISCOVERER = 'disabled'

[pasko, 2016/04/14 12:34:42]

is this used as part of file names? if so, seeing 'disabled' as a file name
would not obviously suggest _what_ is disabled.

So seems this would be better: 'no-discoverer' or 'nocache'.

[gabadie, 2016/04/14 15:43:32]

Done.

+
+# Prefetches everything to load fully from cache (impossible in practice).
+FULLCACHE_DISCOVERER = 'fullcache'

[pasko, 2016/04/14 12:34:42]

nit: prefer words separated by dashes in file names: 'no-cache', 'full-cache'.

This is minor, up to you.

[gabadie, 2016/04/14 15:43:32]

Done.

+
 # Prefetches the first resource following the redirection chain.
 REDIRECTED_MAIN_DISCOVERER = 'redirected-main'
 
 # All resources which are fetched from the main document and their redirections.
-PARSER_DISCOVERER = 'parser',
+PARSER_DISCOVERER = 'parser'
 
 # Simulation of HTMLPreloadScanner on the main document and their redirections.
-HTML_PRELOAD_SCANNER_DISCOVERER = 'html-scanner',
+HTML_PRELOAD_SCANNER_DISCOVERER = 'html-scanner'
 
 SUBRESOURCE_DISCOVERERS = set([
+  DISABLED_DISCOVERER,
+  FULLCACHE_DISCOVERER,
   REDIRECTED_MAIN_DISCOVERER,
   PARSER_DISCOVERER,
   HTML_PRELOAD_SCANNER_DISCOVERER
 ])
 
 
 def PatchWpr(wpr_archive_path):
   """Patches a WPR archive to get all resources into the HTTP cache and avoid
   invalidation and revalidations.
 
@@ skipping 44 matching lines @@
       'unknown prefetch simulation {}'.format(subresource_discoverer)
 
   # Load trace and related infos.
   logging.info('loading %s' % loading_trace_path)
   trace = LoadingTrace.FromJsonFile(loading_trace_path)
   dependencies_lens = RequestDependencyLens(trace)
   first_resource_request = trace.request_track.GetFirstResourceRequest()
 
   # Build the list of discovered requests according to the desired simulation.
   discovered_requests = []
-  if subresource_discoverer == REDIRECTED_MAIN_DISCOVERER:
+  if subresource_discoverer == DISABLED_DISCOVERER:
+    pass
+  elif subresource_discoverer == FULLCACHE_DISCOVERER:
+    discovered_requests = trace.request_track.GetEvents()
+  elif subresource_discoverer == REDIRECTED_MAIN_DISCOVERER:
     discovered_requests = \
         [dependencies_lens.GetRedirectChain(first_resource_request)[-1]]
   elif subresource_discoverer == PARSER_DISCOVERER:
     discovered_requests = PrefetchSimulationView.ParserDiscoverableRequests(
         first_resource_request, dependencies_lens)
   elif subresource_discoverer == HTML_PRELOAD_SCANNER_DISCOVERER:
     discovered_requests = PrefetchSimulationView.PreloadedRequests(
         first_resource_request, dependencies_lens, trace)
   else:
     assert False
 
   # Prune out data:// requests.
   whitelisted_urls = set()
   logging.info('white-listing %s' % first_resource_request.url)
-  whitelisted_urls.add(first_resource_request.url)
   for request in discovered_requests:
     # Work-around where the protocol may be none for an unclear reason yet.
     # TODO(gabadie): Follow up on this with Clovis guys and possibly remove
     #   this work-around.
     if not request.protocol:
       logging.warning('ignoring %s (no protocol)' % request.url)
       continue
     # Ignore data protocols.
     if not request.protocol.startswith('http'):
       continue
     logging.info('white-listing %s' % request.url)
     whitelisted_urls.add(request.url)
   return whitelisted_urls
+
+
+def CompareUrlSet(ref_url_set, url_set, url_set_name, debug_hint='Good luck!'):
+  """Compare URL sets
+
+  Args:
+    ref_url_set: Set of reference urls.
+    url_set: Set of urls to compare to the reference.
+    url_set_name: The set name for logging purposes.
+    debug_hint: A debug hint to help debugging in any case the sets are
+      different.
+  """
+  assert type(ref_url_set) == set
+  assert type(url_set) == set
+  if ref_url_set == url_set:
+    logging.info('  %d %s are matching.' % (len(ref_url_set), url_set_name))
+    return
+  logging.error('  %s are not matching.' % url_set_name)
+  logging.error('    Hint: ' + debug_hint)
+  logging.error('    List of missing resources:')
+  for url in ref_url_set.difference(url_set):
+    logging.error('-     ' + url)
+  logging.error('    List of unexpected resources:')
+  for url in url_set.difference(ref_url_set):
+    logging.error('+     ' + url)
+
+
+def _ListUrlRequests(trace, from_cache=None):
+  urls = set()
+  for request_event in trace.request_track.GetEvents():
+    if request_event.protocol == None:
+      continue
+    if not request_event.protocol.startswith('http'):
+      continue
+    if from_cache is not None and request_event.from_disk_cache != from_cache:
+      continue
+    urls.add(request_event.url)
+  return urls
+
+
+def VerifyBenchmarkOutputDirectory(benchmark_setup_path,
+                                   benchmark_output_directory_path):
+  """Verifies that all run inside the run_output_directory worked as expected.
+
+  Args:
+    benchmark_setup_path: Path of the JSON of the benchmark setup.
+    benchmark_output_directory_path: Path of the benchmark output directory to
+        verify.
+  """
+  benchmark_setup = json.load(open(benchmark_setup_path))
+  cache_whitelist = set(benchmark_setup['cache_whitelist'])
+  url_resources = set(benchmark_setup['url_resources'])
+
+  # Verify requests from traces.
+  run_id = -1
+  while True:
+    run_id += 1
+    run_path = os.path.join(benchmark_output_directory_path, str(run_id))
+    if not os.path.isdir(run_path):
+      break
+    trace_path = os.path.join(run_path, 'trace.json')
+    if not os.path.isfile(trace_path):
+      logging.error('missing trace %s' % trace_path)
+      continue
+    trace = LoadingTrace.FromJsonFile(trace_path)
+    logging.info('verifying %s from %s' % (trace.url, trace_path))
+    CompareUrlSet(url_resources, _ListUrlRequests(trace), 'All resources',
+                  'You may have an issue with an AJAX requests.')
+    CompareUrlSet(url_resources.intersection(cache_whitelist),
+                  _ListUrlRequests(trace, True), 'Cached resources',
+                  'The WPR archive patcher may have an invalidation issue.')
+    CompareUrlSet(url_resources.difference(cache_whitelist),
+                  _ListUrlRequests(trace, False), 'Non cached resources')
+
+
+def ListResourcesUrls(benchmark_output_directory_path):
+  """Lists all requested urls per navigated urls
+
+  Args:
+    benchmark_output_directory_path: Path of the benchmark output directory to
+        verify.
+
+  Returns:
+    {url -> [urls of sub-resources]}
+  """
+  url_subresources = {}
+  run_id = -1
+  while True:
+    run_id += 1
+    run_path = os.path.join(benchmark_output_directory_path, str(run_id))
+    if not os.path.isdir(run_path):
+      break
+    trace_path = os.path.join(run_path, 'trace.json')
+    if not os.path.isfile(trace_path):
+      continue
+    trace = LoadingTrace.FromJsonFile(trace_path)
+    if trace.url in url_subresources:
+      continue
+    logging.info('lists resources of %s from %s' % (trace.url, trace_path))
+    urls_set = set()
+    for request_event in trace.request_track.GetEvents():
+      if not request_event.protocol.startswith('http'):
+        continue
+      if request_event.url not in urls_set:
+        logging.info('  %s' % request_event.url)
+        urls_set.add(request_event.url)
+    url_subresources[trace.url] = [url for url in urls_set]
+  return url_subresources
+
+
+def ValidateCacheArchiveContent(ref_urls, cache_archive_path):
+  """Validates a cache archive content.
+
+  Args:
+    ref_urls: Reference list of urls.
+    cache_archive_path: Cache archive's path to validate.
+  """
+  logging.info('lists cached urls from %s' % cache_archive_path)
+  with common_util.TemporaryDirectory() as cache_directory:
+    chrome_cache.UnzipDirectoryContent(cache_archive_path, cache_directory)
+    cached_urls = \
+        chrome_cache.CacheBackend(cache_directory, 'simple').ListKeys()
+  CompareUrlSet(set(ref_urls), set(cached_urls), 'cached resources',
+                debug_hint='Looks like a response header needs to be patched.')