sandwich: Fixes two sources of KeyError task failures

tools/android/loading/sandwich_prefetch.py

Index: tools/android/loading/sandwich_prefetch.py
diff --git a/tools/android/loading/sandwich_prefetch.py b/tools/android/loading/sandwich_prefetch.py
index d82b7fc88193e3ea0ec74ff415c5302316cd81b8..f78d08f8009b6182190a5e4d15475e5b65717317 100644
--- a/tools/android/loading/sandwich_prefetch.py
+++ b/tools/android/loading/sandwich_prefetch.py
@@ -30,6 +30,7 @@ from prefetch_view import PrefetchSimulationView
 from request_dependencies_lens import RequestDependencyLens
 import sandwich_metrics
 import sandwich_runner
+import sandwich_utils
 import task_manager
 import wpr_backend
 
@@ -199,7 +200,15 @@ def _PruneOutOriginalNoStoreRequests(original_headers_path, requests):
     original_headers = json.load(file_input)
   pruned_requests = set()
   for request in requests:
-    request_original_headers = original_headers[request.url]
+    url = sandwich_utils.NormalizeUrl(request.url)
+    if url not in original_headers:
+      # TODO(gabadie): Dig why these requests were not in WPR.

[pasko, 2016/07/04 15:55:19]

nit: s/Dig/Investigate/

yeah, that's a bit surprising, and suggests to me that our scanner discoverer is
more magic than it should be :(

[gabadie, 2016/07/04 17:03:43]

Done.

+      assert request.failed
+      logging.warning(
+          'could not find original headers for: %s (failure: %s)',
+          url, request.error_text)
+      continue
+    request_original_headers = original_headers[url]
     if ('cache-control' in request_original_headers and
         'no-store' in request_original_headers['cache-control'].lower()):
       pruned_requests.add(request)
@@ -479,6 +488,7 @@ def _ProcessRunOutputDir(
     served_from_network_bytes = 0
     served_from_cache_bytes = 0
     urls_hitting_network = set()
+    response_sizes = {}
     for request in _FilterOutDataAndIncompleteRequests(
         trace.request_track.GetEvents()):
       # Ignore requests served from the blink's cache.
@@ -486,9 +496,18 @@ def _ProcessRunOutputDir(
         continue
       urls_hitting_network.add(request.url)
       if request.from_disk_cache:
-        served_from_cache_bytes += cached_encoded_data_lengths[request.url]
+        if request.url in cached_encoded_data_lengths:
+          response_size = cached_encoded_data_lengths[request.url]
+        else:
+          # Some fat webpages may overflow the Memory cache, and so some
+          # requests might be served from disk cache couple of times per page
+          # load.

[pasko, 2016/07/04 15:55:19]

can you spit a log message here? It would be nice to make a list of URLs that
overflow memory cache - for folks who are busy reimplementing the memorycache.

[gabadie, 2016/07/04 17:03:43]

Done.

+          response_size = response_sizes[request.url]
+        served_from_cache_bytes += response_size
       else:
-        served_from_network_bytes += request.GetEncodedDataLength()
+        response_size = request.GetEncodedDataLength()
+        served_from_network_bytes += response_size
+      response_sizes[request.url] = response_size
 
     # Make sure the served from blink's cache requests have at least one
     # corresponding request that was not served from the blink's cache.
@@ -572,6 +591,14 @@ class PrefetchBenchmarkBuilder(task_manager.Builder):
       # Save up original response headers.
       original_response_headers = {e.url: e.GetResponseHeadersDict() \
           for e in wpr_archive.ListUrlEntries()}
+      logging.info('save up response headers for %d resources',
+                   len(original_response_headers))
+      if not original_response_headers:
+        # TODO(gabadie): How is it possible to not even have the main resource
+        # in the WPR archive?

[pasko, 2016/07/04 15:55:18]

please link to the bug (and preferably the comment # in the bug) here that lists
the URL that can trigger this.

... otherwise this TODO will be harder than necessary to address in the future.

[gabadie, 2016/07/04 17:03:43]

Done.

+        raise sandwich_utils.SandwichKnownError(
+            'Looks like no resources were recorded in WPR during: {}'.format(
+                self._common_builder.original_wpr_task.name))
       with open(self._original_headers_path, 'w') as file_output:
         json.dump(original_response_headers, file_output)