sandwich: Use cachetool's batch mode to speed-up cache processing.

tools/android/loading/sandwich_prefetch.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 Implements a task builder for benchmarking effects of NoState Prefetch.
 Noticeable steps of the task pipeline:
   * Save a WPR archive
   * Process the WPR archive to make all resources cacheable
   * Process cache archive to patch response headers back to their original
@@ skipping 98 matching lines @@
 
 
 def _FilterOutDataAndIncompleteRequests(requests):
   for request in filter(lambda r: not r.IsDataRequest(), requests):
     # The protocol is only known once the response has been received. But the
     # trace recording might have been stopped with still some JavaScript
     # originated requests that have not received any responses yet.
     if request.protocol is None:
       assert not request.HasReceivedResponse()
       continue
-    if request.protocol == 'about':
+    if request.protocol in {'about', 'blob'}:
       continue
     if request.protocol not in {'http/0.9', 'http/1.0', 'http/1.1'}:
       raise RuntimeError('Unknown request protocol {}'.format(request.protocol))
     yield request
 
 
 def _PatchCacheArchive(cache_archive_path, loading_trace_path,
                        cache_archive_dest_path):
   """Patch the cache archive.
 
   Note: This method update the raw response headers of cache entries' to store
     the ones such as Set-Cookie that were pruned by the
     net::HttpCacheTransaction, and remove the stream index 2 holding resource's
     compile meta data.
 
   Args:
     cache_archive_path: Input archive's path to patch.
     loading_trace_path: Path of the loading trace that have recorded the cache
         archive <cache_archive_path>.
     cache_archive_dest_path: Archive destination's path.
   """
+  logging.info('loading trace: %s', loading_trace_path)
   trace = loading_trace.LoadingTrace.FromJsonFile(loading_trace_path)
   with common_util.TemporaryDirectory(prefix='sandwich_tmp') as tmp_path:
     cache_path = os.path.join(tmp_path, 'cache')
     chrome_cache.UnzipDirectoryContent(cache_archive_path, cache_path)
-    cache_backend = chrome_cache.CacheBackend(cache_path, 'simple')
+    cache_backend = chrome_cache.BatchCacheBackend(
+        cache_path, chrome_cache.CacheBackendType.SIMPLE)
     cache_entries = set(cache_backend.ListKeys())
     logging.info('Original cache size: %d bytes' % cache_backend.GetSize())
     for request in _FilterOutDataAndIncompleteRequests(
         trace.request_track.GetEvents()):
       # On requests having an upload data stream such as POST requests,
       # net::HttpCache::GenerateCacheKey() prefixes the cache entry's key with
       # the upload data stream's session unique identifier.
       #
       # It is fine to not patch these requests since when reopening Chrome,
       # there is no way the entry can be reused since the upload data stream's
       # identifier will be different.
       #
       # The fact that these entries are kept in the cache after closing Chrome
-      # properly by closing the Chrome tab as the ChromeControler.SetSlowDeath()
-      # do is known chrome bug (crbug.com/610725).
+      # properly by closing the Chrome tab as the
+      # ChromeControler.SetSlowDeath() do is known chrome bug
+      # (crbug.com/610725).
       if request.url not in cache_entries:
         continue
       # Chrome prunes Set-Cookie from response headers before storing them in
-      # disk cache. Also, it adds implicit "Vary: cookie" header to all redirect
-      # response headers. Sandwich manages the cache, but between recording the
-      # cache and benchmarking the cookie jar is invalidated. This leads to
-      # invalidation of all cacheable redirects.
+      # disk cache. Also, it adds implicit "Vary: cookie" header to all
+      # redirect response headers. Sandwich manages the cache, but between
+      # recording the # cache and benchmarking the cookie jar is invalidated.
+      # This leads to invalidation of all cacheable redirects.
       raw_headers = request.GetRawResponseHeaders()
       cache_backend.UpdateRawResponseHeaders(request.url, raw_headers)
       # NoState-Prefetch would only fetch the resources, but not parse them.
       cache_backend.DeleteStreamForKey(request.url, 2)
+    cache_backend.ProcessBatch()
+    logging.info('Patched cache size: %d bytes' % cache_backend.GetSize())
     chrome_cache.ZipDirectoryContent(cache_path, cache_archive_dest_path)
-    logging.info('Patched cache size: %d bytes' % cache_backend.GetSize())
 
 
 def _DiscoverRequests(dependencies_lens, subresource_discoverer):
   trace = dependencies_lens.loading_trace
   first_resource_request = trace.request_track.GetFirstResourceRequest()
 
   if subresource_discoverer == Discoverer.EmptyCache:
     requests = []
   elif subresource_discoverer == Discoverer.FullCache:
     requests = dependencies_lens.loading_trace.request_track.GetEvents()
@@ skipping 212 matching lines @@
         {URL of all requests: encoded_data_length},
       'effective_post_requests': [URLs of POST requests],
       'expected_cached_resources': [URLs of resources expected to be cached],
       'successfully_cached': [URLs of cached sub-resources]
     }
   """
   # TODO(gabadie): What's the best way of propagating errors happening in here?
   logging.info('lists cached urls from %s' % cache_archive_path)
   with common_util.TemporaryDirectory() as cache_directory:
     chrome_cache.UnzipDirectoryContent(cache_archive_path, cache_directory)
-    cache_keys = set(
-        chrome_cache.CacheBackend(cache_directory, 'simple').ListKeys())
+    cache_keys = set(chrome_cache.CacheBackend(
+        cache_directory, chrome_cache.CacheBackendType.SIMPLE).ListKeys())
   trace = loading_trace.LoadingTrace.FromJsonFile(cache_build_trace_path)
   effective_requests = _ListUrlRequests(trace, _RequestOutcome.All)
   effective_post_requests = _ListUrlRequests(trace, _RequestOutcome.Post)
   effective_encoded_data_lengths = {}
   for request in _FilterOutDataAndIncompleteRequests(
       trace.request_track.GetEvents()):
     if request.from_disk_cache or request.served_from_cache:
       # At cache archive creation time, a request might be loaded several times,
       # but avoid the request.encoded_data_length == 0 if loaded from cache.
       continue
@@ skipping 280 matching lines @@
       run_metrics_list = _ProcessRunOutputDir(
           cache_validation_result, benchmark_setup, RunBenchmark.path)
       with open(ProcessRunOutputDir.path, 'w') as csv_file:
         writer = csv.DictWriter(csv_file, fieldnames=(additional_column_names +
                                     sandwich_metrics.COMMON_CSV_COLUMN_NAMES))
         writer.writeheader()
         for trace_metrics in run_metrics_list:
           writer.writerow(trace_metrics)
 
     self._common_builder.default_final_tasks.append(ProcessRunOutputDir)