sandwich: Implements filter-cache sub-command.

tools/android/loading/chrome_cache.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Takes care of manipulating the chrome's HTTP cache.
 """
 
 from datetime import datetime
 import json
 import os
+import shutil
 import subprocess
 import sys
 import tempfile
 import zipfile
 
 _SRC_DIR = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..', '..'))
 
 sys.path.append(os.path.join(_SRC_DIR, 'build', 'android'))
 from pylib import constants
@@ skipping 113 matching lines @@
 
 
 def ZipDirectoryContent(root_directory_path, archive_dest_path):
   """Zip a directory's content recursively with all the directories'
   timestamps preserved.
 
   Args:
     root_directory_path: The directory's path to archive.
     archive_dest_path: Archive destination's path.
   """
+  if os.path.isfile(archive_dest_path):

[mattcary, 2016/02/26 09:33:08]

This doesn't seem to be necessary---zipfile will silently overwrite existing
files as far as I can tell. Seems harmless enough, though, even if my instinct
is always to remove code.

[pasko, 2016/02/26 17:10:06]

is it really necessary to remove before writing? According to the code below we
are not expecting a file to be there, so maybe this code is dead and not needed?

[gabadie, 2016/03/01 10:40:48]

Done.

+    os.remove(archive_dest_path)
   with zipfile.ZipFile(archive_dest_path, 'w') as zip_output:
     timestamps = {}
     root_directory_stats = os.stat(root_directory_path)
     timestamps['.'] = {
         'atime': root_directory_stats.st_atime,
         'mtime': root_directory_stats.st_mtime}
     for directory_path, dirnames, filenames in os.walk(root_directory_path):
       for dirname in dirnames:
         subdirectory_path = os.path.join(directory_path, dirname)
         subdirectory_relative_path = os.path.relpath(subdirectory_path,
@@ skipping 117 matching lines @@
         self._cache_directory_path,
         self._cache_backend_type,
         operation]
     editor_tool_cmd.extend(args)
     process = subprocess.Popen(editor_tool_cmd, stdout=subprocess.PIPE)
     stdout_data, _ = process.communicate()
     assert process.returncode == 0
     return stdout_data
 
 
+def ApplyUrlWhitelistToCacheArchive(cache_archive_path,
+                                    whitelisted_urls,
+                                    output_cache_archive_path):
+  """Generate a new cache archive containing only whitelisted urls.
+
+  Args:
+    cache_archive_path: Path of the cache archive to apply the white listing.
+    whitelisted_urls: Set of url to keep in cache.
+    output_cache_archive_path: Destination path of cache archive containing only
+      white-listed urls.
+  """
+  cache_temp_directory = tempfile.mkdtemp(suffix='.cache')
+  UnzipDirectoryContent(cache_archive_path, cache_temp_directory)
+  backend = CacheBackend(cache_temp_directory, 'simple')
+  cached_urls = backend.ListKeys()
+  for cached_url in cached_urls:
+    if cached_url not in whitelisted_urls:
+      backend.DeleteKey(cached_url)
+  for cached_url in backend.ListKeys():
+    assert cached_url in whitelisted_urls
+  ZipDirectoryContent(cache_temp_directory, output_cache_archive_path)
+  shutil.rmtree(cache_temp_directory)

[pasko, 2016/02/26 17:10:06]

the try..finally would avoid polluting the filesystem

[gabadie, 2016/03/01 10:40:48]

Good point. I think we should just do a util like
https://docs.python.org/3/library/tempfile.html#tempfile.TemporaryDirectory.

Done.

+
+
 if __name__ == '__main__':
   import argparse

[pasko, 2016/02/26 17:10:06]

why import here?

[gabadie, 2016/03/01 10:40:48]

Because this is a lib manual test (in the mean time of the integrated test for
sandwich). So we don't want to pollute the module import list when including
into another module.

[pasko, 2016/03/01 16:01:11]

isn't this against the style guide? To avoid the pollution please prefer to
create a separate file for manual testing.

[gabadie, 2016/03/01 16:49:09]

According to Benoit, that is a common practice.
https://code.google.com/p/chromium/codesearch#chromium/src/tools/android/load...

[pasko, 2016/03/01 17:55:53]

I disagree with Benoit here. In the Chromium codebase the only occurrences of
this are in tools/android/loading. And in this one weird place:
https://goo.gl/SozSvM

We should not do it.

   parser = argparse.ArgumentParser(description='Tests cache back-end.')

[pasko, 2016/02/26 17:10:06]

nit: backend in one word

[gabadie, 2016/03/01 10:40:48]

Looks like the two are correct. Keeping it to be consistent within this
directory.

[pasko, 2016/03/01 16:01:11]

They are both correct, but we should prefer using it as "backend" to be
consistent with the rest of Chrome.

Please also see Don Knuth's opinion on email vs e-mail:
http://www-cs-faculty.stanford.edu/~uno/email.html

But this is a nit for now, so you can leave it unchanged for now

[gabadie, 2016/03/01 16:49:09]

Acknowledged.

   parser.add_argument('cache_path', type=str)
   parser.add_argument('backend_type', type=str, choices=BACKEND_TYPES)
   command_line_args = parser.parse_args()
 
   cache_backend = CacheBackend(
       cache_directory_path=command_line_args.cache_path,
       cache_backend_type=command_line_args.backend_type)
   keys = cache_backend.ListKeys()
   print '{}\'s HTTP response header:'.format(keys[0])
   print cache_backend.GetStreamForKey(keys[0], 0)
   cache_backend.DeleteKey(keys[1])
   assert keys[1] not in cache_backend.ListKeys()