isolateserver: DiskCache format v2

client/tests/isolateserver_test.py

 #!/usr/bin/env python
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 # pylint: disable=W0212,W0223,W0231,W0613
 
 import base64
 import collections
 import hashlib
@@ skipping 1230 matching lines @@
     # Max: 100 bytes, 2 items
     # Min free disk: 1000 bytes.
     self._policies = isolateserver.CachePolicies(100, 1000, 2)
     def get_free_space(p):
       self.assertEqual(p, self.tempdir)
       return self._free_disk
     self.mock(file_path, 'get_free_space', get_free_space)
     # TODO(maruel): Test the following.
     #cache.touch()
 
+    self.now = 0.0
+
   def get_cache(self):
-    return isolateserver.DiskCache(self.tempdir, self._policies, self._algo)
+    return isolateserver.DiskCache(
+        self.tempdir, self._policies, self._algo, time_fn=lambda: self.now)
 
   def to_hash(self, content):
     return self._algo(content).hexdigest(), content
 
   def test_read_evict(self):
     self._free_disk = 1100
     h_a = self.to_hash('a')[0]
     with self.get_cache() as cache:
       cache.write(h_a, 'a')
       with cache.getfileobj(h_a) as f:
@@ skipping 23 matching lines @@
     with self.assertRaises(isolateserver.Error):
       cache.write(*self.to_hash('e'))
 
   def test_cleanup(self):
     # Inject an item without a state.json. It will be deleted on cleanup.
     h_a = self.to_hash('a')[0]
     isolateserver.file_write(os.path.join(self.tempdir, h_a), 'a')
     cache = self.get_cache()
     self.assertEqual([], sorted(cache._lru._items.iteritems()))
     self.assertEqual(
-        sorted([h_a, u'state.json']), sorted(os.listdir(self.tempdir)))
+        sorted([h_a, u'state.json', u'VERSION']), sorted(os.listdir(self.tempdir)))
     cache.cleanup()
-    self.assertEqual([u'state.json'], os.listdir(self.tempdir))
+    self.assertEqual([u'state.json', u'VERSION'], os.listdir(self.tempdir))
 
   def test_policies_active_trimming(self):
     # Start with a larger cache, add many object.
     # Reload the cache with smaller policies, the cache should be trimmed on
     # load.
     h_a = self.to_hash('a')[0]
     h_b = self.to_hash('b')[0]
     h_c = self.to_hash('c')[0]
     h_large, large = self.to_hash('b' * 99)
 
     # Max policies is 100 bytes, 2 items, 1000 bytes free space.
     self._free_disk = 1101
     with self.get_cache() as cache:
       cache.write(h_a, 'a')
       cache.write(h_large, large)
       # Cache (size and # items) is not enforced while adding items. The
       # rationale is that a task may request more data than the size of the
       # cache policies. As long as there is free space, this is fine.
       cache.write(h_b, 'b')
       expected = sorted(((h_a, 1), (h_large, len(large)), (h_b, 1)))
-      self.assertEqual(expected, sorted(cache._lru._items.iteritems()))
+      self.assertEqual(expected, sorted(cache._sizes()))
       self.assertEqual(h_a, cache._protected)
       self.assertEqual(1000, cache._free_disk)
       self.assertEqual(0, cache.initial_number_items)
       self.assertEqual(0, cache.initial_size)
       # Free disk is enforced, because otherwise we assume the task wouldn't
       # be able to start. In this case, it throws an exception since all items
       # are protected. The item is added since it's detected after the fact.
       with self.assertRaises(isolateserver.Error):
         cache.write(h_c, 'c')
 
     # At this point, after the implicit trim in __exit__(), h_a and h_large were
     # evicted.
     self.assertEqual(
-        sorted([h_b, h_c, u'state.json']), sorted(os.listdir(self.tempdir)))
+        set([h_b[:2], h_c[:2], u'state.json', u'VERSION']),
+        set(os.listdir(self.tempdir)))
 
     # Allow 3 items and 101 bytes so h_large is kept.
     self._policies = isolateserver.CachePolicies(101, 1000, 3)
     with self.get_cache() as cache:
       cache.write(h_large, large)
       self.assertEqual(2, cache.initial_number_items)
       self.assertEqual(2, cache.initial_size)
 
     self.assertEqual(
-        sorted([h_b, h_c, h_large, u'state.json']),
-        sorted(os.listdir(self.tempdir)))
+        set([h_b[:2], h_c[:2], h_large[:2], u'state.json', u'VERSION']),
+        set(os.listdir(self.tempdir)))
 
     # Assert that trimming is done in constructor too.
     self._policies = isolateserver.CachePolicies(100, 1000, 2)
     with self.get_cache() as cache:
-      expected = collections.OrderedDict([(h_c, 1), (h_large, len(large))])
-      self.assertEqual(expected, cache._lru._items)
+      self.assertEqual(
+          [(h_c, 1), (h_large, len(large))],
+          list(cache._sizes()))
       self.assertEqual(None, cache._protected)
       self.assertEqual(1101, cache._free_disk)
       self.assertEqual(2, cache.initial_number_items)
       self.assertEqual(100, cache.initial_size)
 
+  def test_timestamp(self):
+    self._free_disk = 1100
+    cache = self.get_cache()
+    with cache:
+      self.now = 1
+      cache.write('deadbeef', '1')
+
+      self.now = 2
+      cache.write('badcoffee', '2')
+
+    self.assertEqual(
+      [
+        ('deadbeef', [1, 1]),
+        ('badcoffee', [1, 2]),
+      ],
+      cache._lru._items.items()
+    )
+
+    loaded = self.get_cache()
+    self.assertEqual(cache._lru._items, loaded._lru._items)
+
 
 def clear_env_vars():
   for e in ('ISOLATE_DEBUG', 'ISOLATE_SERVER'):
     os.environ.pop(e, None)
 
 
 if __name__ == '__main__':
   fix_encoding.fix_encoding()
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
   logging.basicConfig(
       level=(logging.DEBUG if '-v' in sys.argv else logging.CRITICAL))
   clear_env_vars()
   unittest.main()