Docserver: Override Walk in CachingFileSystem

chrome/common/extensions/docs/server2/caching_file_system_test.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import os
 import sys
 import unittest
 
 from caching_file_system import CachingFileSystem
@@ skipping 40 matching lines @@
       './test3.txt': 'test3\n',
     }
     self.assertEqual(
         expected,
         file_system.Read(['./test1.txt', './test2.txt', './test3.txt']).Get())
 
   def testListDir(self):
     file_system = self._CreateCachingFileSystem(
         _CreateLocalFs(), start_empty=False)
     expected = ['dir/'] + ['file%d.html' % i for i in range(7)]
-    file_system._read_object_store.Set(
+    file_system._read_cache.Set(
         'list/',
         (expected, file_system.Stat('list/').version))
     self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
 
     expected.remove('file0.html')
-    file_system._read_object_store.Set(
+    file_system._read_cache.Set(
         'list/',
         (expected, file_system.Stat('list/').version))
     self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
 
   def testCaching(self):
     test_fs = TestFileSystem({
       'bob': {
         'bob0': 'bob/bob0 contents',
         'bob1': 'bob/bob1 contents',
         'bob2': 'bob/bob2 contents',
@@ skipping 27 matching lines @@
     # Test if there is a newer version, the resource is re-fetched.
     file_system = create_empty_caching_fs()
     test_fs.IncrementStat();
     future = file_system.ReadSingle('bob/bob0')
     self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
     self.assertEqual('bob/bob0 contents', future.Get())
     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1))
 
     # Test directory and subdirectory stats are cached.
     file_system = create_empty_caching_fs()
-    file_system._stat_object_store.Del('bob/bob0')
-    file_system._read_object_store.Del('bob/bob0')
-    file_system._stat_object_store.Del('bob/bob1')
+    file_system._stat_cache.Del('bob/bob0')
+    file_system._read_cache.Del('bob/bob0')
+    file_system._stat_cache.Del('bob/bob1')
     test_fs.IncrementStat();
     futures = (file_system.ReadSingle('bob/bob1'),
                file_system.ReadSingle('bob/bob0'))
     self.assertTrue(*mock_fs.CheckAndReset(read_count=2))
     self.assertEqual(('bob/bob1 contents', 'bob/bob0 contents'),
                      tuple(future.Get() for future in futures))
     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=2, stat_count=1))
     self.assertEqual('bob/bob1 contents',
                      file_system.ReadSingle('bob/bob1').Get())
     self.assertTrue(*mock_fs.CheckAndReset())
 
     # Test a more recent parent directory doesn't force a refetch of children.
     file_system = create_empty_caching_fs()
-    file_system._read_object_store.Del('bob/bob0')
-    file_system._read_object_store.Del('bob/bob1')
+    file_system._read_cache.Del('bob/bob0')
+    file_system._read_cache.Del('bob/bob1')
     futures = (file_system.ReadSingle('bob/bob1'),
                file_system.ReadSingle('bob/bob2'),
                file_system.ReadSingle('bob/bob3'))
     self.assertTrue(*mock_fs.CheckAndReset(read_count=3))
     self.assertEqual(
         ('bob/bob1 contents', 'bob/bob2 contents', 'bob/bob3 contents'),
         tuple(future.Get() for future in futures))
     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=3, stat_count=1))
 
     test_fs.IncrementStat(path='bob/bob0')
     file_system = create_empty_caching_fs()
     self.assertEqual('bob/bob1 contents',
                      file_system.ReadSingle('bob/bob1').Get())
     self.assertEqual('bob/bob2 contents',
                      file_system.ReadSingle('bob/bob2').Get())
     self.assertEqual('bob/bob3 contents',
                      file_system.ReadSingle('bob/bob3').Get())
     self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
 
     file_system = create_empty_caching_fs()
-    file_system._stat_object_store.Del('bob/bob0')
+    file_system._stat_cache.Del('bob/bob0')
     future = file_system.ReadSingle('bob/bob0')
     self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
     self.assertEqual('bob/bob0 contents', future.Get())
     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1))
     self.assertEqual('bob/bob0 contents',
                      file_system.ReadSingle('bob/bob0').Get())
     self.assertTrue(*mock_fs.CheckAndReset())
 
     # Test skip_not_found caching behavior.
     file_system = create_empty_caching_fs()
@@ skipping 80 matching lines @@
       }
     }))
     def read_skip_not_found(paths):
       return caching_fs.Read(paths, skip_not_found=True).Get()
     self.assertEqual({}, read_skip_not_found(('grub',)))
     self.assertEqual({}, read_skip_not_found(('bob/bob2',)))
     self.assertEqual({
       'bob/bob0': 'bob/bob0 contents',
     }, read_skip_not_found(('bob/bob0', 'bob/bob2')))
 
+  def testWalkCaching(self):
+    test_fs = TestFileSystem({
+      'root': {
+        'file1': 'file1',
+        'file2': 'file2',
+        'dir1': {
+          'dir1_file1': 'dir1_file1',
+          'dir2': {},
+          'dir3': {
+            'dir3_file1': 'dir3_file1',
+            'dir3_file2': 'dir3_file2'
+          }
+        }
+      }
+    })
+    mock_fs = MockFileSystem(test_fs)
+    file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
+    for walkinfo in file_system.Walk(''):
+      pass
+    self.assertTrue(*mock_fs.CheckAndReset(
+        read_resolve_count=5, read_count=5, stat_count=5))
+
+    all_dirs, all_files = [], []
+    for root, dirs, files in file_system.Walk(''):
+      all_dirs.extend(dirs)
+      all_files.extend(files)
+    self.assertEqual(sorted(['root/', 'dir1/', 'dir2/', 'dir3/']),
+                     sorted(all_dirs))
+    self.assertEqual(
+        sorted(['file1', 'file2', 'dir1_file1', 'dir3_file1', 'dir3_file2']),
+        sorted(all_files))
+    # All data should be cached.
+    self.assertTrue(*mock_fs.CheckAndReset())
+
+    # Starting from a different root should still pull cached data.
+    for walkinfo in file_system.Walk('root/dir1/'):
+      pass
+    self.assertTrue(*mock_fs.CheckAndReset())

[not at google - send to devlin, 2014/08/29 20:26:07]

Could you also test with a new CachingFileSystem? Calling
_CreateCachingFileSystem multiple times from the same test will result in the
same object store being returned, so I'd expect a Stat of 1 each time. Or maybe
5 each time. Ideally 1 because Walk could be smart enough to know that the file
system hasn't changed by Stat-ing the root. Maybe StatAsync already has this
optimisation and it's not worth worrying about.

Or you can TODO-ify this. If it still gets a 30% improvement we should submit it
now.

 
 if __name__ == '__main__':
   unittest.main()