sandwich: Use cachetool's batch mode to speed-up cache processing.

tools/android/loading/sandwich_prefetch.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 Implements a task builder for benchmarking effects of NoState Prefetch.
 Noticeable steps of the task pipeline:
   * Save a WPR archive
   * Process the WPR archive to make all resources cacheable
   * Process cache archive to patch response headers back to their original
@@ skipping 98 matching lines @@
 
 
 def _FilterOutDataAndIncompleteRequests(requests):
   for request in filter(lambda r: not r.IsDataRequest(), requests):
     # The protocol is only known once the response has been received. But the
     # trace recording might have been stopped with still some JavaScript
     # originated requests that have not received any responses yet.
     if request.protocol is None:
       assert not request.HasReceivedResponse()
       continue
-    if request.protocol == 'about':
+    if request.protocol in {'about', 'blob'}:
       continue
     if request.protocol not in {'http/0.9', 'http/1.0', 'http/1.1'}:
       raise RuntimeError('Unknown request protocol {}'.format(request.protocol))
     yield request
 
 
 def _PatchCacheArchive(cache_archive_path, loading_trace_path,
                        cache_archive_dest_path):
   """Patch the cache archive.
 
   Note: This method update the raw response headers of cache entries' to store
     the ones such as Set-Cookie that were pruned by the
     net::HttpCacheTransaction, and remove the stream index 2 holding resource's
     compile meta data.
 
   Args:
     cache_archive_path: Input archive's path to patch.
     loading_trace_path: Path of the loading trace that have recorded the cache
         archive <cache_archive_path>.
     cache_archive_dest_path: Archive destination's path.
   """
+  logging.info('loading trace: %s', loading_trace_path)
   trace = loading_trace.LoadingTrace.FromJsonFile(loading_trace_path)
   with common_util.TemporaryDirectory(prefix='sandwich_tmp') as tmp_path:
     cache_path = os.path.join(tmp_path, 'cache')
     chrome_cache.UnzipDirectoryContent(cache_archive_path, cache_path)
-    cache_backend = chrome_cache.CacheBackend(cache_path, 'simple')
-    cache_entries = set(cache_backend.ListKeys())
-    logging.info('Original cache size: %d bytes' % cache_backend.GetSize())
-    for request in _FilterOutDataAndIncompleteRequests(
-        trace.request_track.GetEvents()):
-      # On requests having an upload data stream such as POST requests,
-      # net::HttpCache::GenerateCacheKey() prefixes the cache entry's key with
-      # the upload data stream's session unique identifier.
-      #
-      # It is fine to not patch these requests since when reopening Chrome,
-      # there is no way the entry can be reused since the upload data stream's
-      # identifier will be different.
-      #
-      # The fact that these entries are kept in the cache after closing Chrome
-      # properly by closing the Chrome tab as the ChromeControler.SetSlowDeath()
-      # do is known chrome bug (crbug.com/610725).
-      if request.url not in cache_entries:
-        continue
-      # Chrome prunes Set-Cookie from response headers before storing them in
-      # disk cache. Also, it adds implicit "Vary: cookie" header to all redirect
-      # response headers. Sandwich manages the cache, but between recording the
-      # cache and benchmarking the cookie jar is invalidated. This leads to
-      # invalidation of all cacheable redirects.
-      raw_headers = request.GetRawResponseHeaders()
-      cache_backend.UpdateRawResponseHeaders(request.url, raw_headers)
-      # NoState-Prefetch would only fetch the resources, but not parse them.
-      cache_backend.DeleteStreamForKey(request.url, 2)
+    with chrome_cache.OnlineCacheBackend(
+        cache_path, chrome_cache.CacheBackendType.Simple) as cache_backend:
+      cache_entries = set(cache_backend.ListKeys())
+      logging.info('Original cache size: %d bytes' % cache_backend.GetSize())
+      for request in _FilterOutDataAndIncompleteRequests(
+          trace.request_track.GetEvents()):
+        # On requests having an upload data stream such as POST requests,
+        # net::HttpCache::GenerateCacheKey() prefixes the cache entry's key with
+        # the upload data stream's session unique identifier.
+        #
+        # It is fine to not patch these requests since when reopening Chrome,
+        # there is no way the entry can be reused since the upload data stream's
+        # identifier will be different.
+        #
+        # The fact that these entries are kept in the cache after closing Chrome
+        # properly by closing the Chrome tab as the
+        # ChromeControler.SetSlowDeath() do is known chrome bug
+        # (crbug.com/610725).
+        if request.url not in cache_entries:
+          continue
+        # Chrome prunes Set-Cookie from response headers before storing them in
+        # disk cache. Also, it adds implicit "Vary: cookie" header to all
+        # redirect response headers. Sandwich manages the cache, but between
+        # recording the # cache and benchmarking the cookie jar is invalidated.
+        # This leads to invalidation of all cacheable redirects.
+        raw_headers = request.GetRawResponseHeaders()
+        try:
+          cache_backend.UpdateRawResponseHeaders(request.url, raw_headers)
+          # NoState-Prefetch would only fetch the resources, but not parse them.
+          cache_backend.DeleteStreamForKey(request.url, 2)
+          # Sync operations to actually catch errors here.
+          cache_backend.Sync()
+        except chrome_cache.CacheBackendError as error:
+          # For some reason, sometimes the cachetool can't find key's entry when
+          # not using the online mode.
+          logging.warning('cachetool error: %s', repr(error))
+      logging.info('Patched cache size: %d bytes' % cache_backend.GetSize())
     chrome_cache.ZipDirectoryContent(cache_path, cache_archive_dest_path)
-    logging.info('Patched cache size: %d bytes' % cache_backend.GetSize())
 
 
 def _DiscoverRequests(dependencies_lens, subresource_discoverer):
   trace = dependencies_lens.loading_trace
   first_resource_request = trace.request_track.GetFirstResourceRequest()
 
   if subresource_discoverer == Discoverer.EmptyCache:
     requests = []
   elif subresource_discoverer == Discoverer.FullCache:
     requests = dependencies_lens.loading_trace.request_track.GetEvents()
@@ skipping 212 matching lines @@
         {URL of all requests: encoded_data_length},
       'effective_post_requests': [URLs of POST requests],
       'expected_cached_resources': [URLs of resources expected to be cached],
       'successfully_cached': [URLs of cached sub-resources]
     }
   """
   # TODO(gabadie): What's the best way of propagating errors happening in here?
   logging.info('lists cached urls from %s' % cache_archive_path)
   with common_util.TemporaryDirectory() as cache_directory:
     chrome_cache.UnzipDirectoryContent(cache_archive_path, cache_directory)
-    cache_keys = set(
-        chrome_cache.CacheBackend(cache_directory, 'simple').ListKeys())
+    cache_keys = set(chrome_cache.CacheBackend(
+        cache_directory, chrome_cache.CacheBackendType.Simple).ListKeys())
   trace = loading_trace.LoadingTrace.FromJsonFile(cache_build_trace_path)
   effective_requests = _ListUrlRequests(trace, _RequestOutcome.All)
   effective_post_requests = _ListUrlRequests(trace, _RequestOutcome.Post)
   effective_encoded_data_lengths = {}
   for request in _FilterOutDataAndIncompleteRequests(
       trace.request_track.GetEvents()):
     if request.from_disk_cache or request.served_from_cache:
       # At cache archive creation time, a request might be loaded several times,
       # but avoid the request.encoded_data_length == 0 if loaded from cache.
       continue
@@ skipping 280 matching lines @@
       run_metrics_list = _ProcessRunOutputDir(
           cache_validation_result, benchmark_setup, RunBenchmark.path)
       with open(ProcessRunOutputDir.path, 'w') as csv_file:
         writer = csv.DictWriter(csv_file, fieldnames=(additional_column_names +
                                     sandwich_metrics.COMMON_CSV_COLUMN_NAMES))
         writer.writeheader()
         for trace_metrics in run_metrics_list:
           writer.writerow(trace_metrics)
 
     self._common_builder.default_final_tasks.append(ProcessRunOutputDir)