Docserver: Override Walk in CachingFileSystem

chrome/common/extensions/docs/server2/caching_file_system_test.py

Index: chrome/common/extensions/docs/server2/caching_file_system_test.py
diff --git a/chrome/common/extensions/docs/server2/caching_file_system_test.py b/chrome/common/extensions/docs/server2/caching_file_system_test.py
index 78becf1a8dc9c8e35f567662cdae3b73dc266bda..5cd7504e6c5aa5edbfe6c00a36b20bd3bd12270b 100755
--- a/chrome/common/extensions/docs/server2/caching_file_system_test.py
+++ b/chrome/common/extensions/docs/server2/caching_file_system_test.py
@@ -58,13 +58,13 @@ class CachingFileSystemTest(unittest.TestCase):
     file_system = self._CreateCachingFileSystem(
         _CreateLocalFs(), start_empty=False)
     expected = ['dir/'] + ['file%d.html' % i for i in range(7)]
-    file_system._read_object_store.Set(
+    file_system._read_cache.Set(
         'list/',
         (expected, file_system.Stat('list/').version))
     self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
 
     expected.remove('file0.html')
-    file_system._read_object_store.Set(
+    file_system._read_cache.Set(
         'list/',
         (expected, file_system.Stat('list/').version))
     self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
@@ -112,9 +112,9 @@ class CachingFileSystemTest(unittest.TestCase):
 
     # Test directory and subdirectory stats are cached.
     file_system = create_empty_caching_fs()
-    file_system._stat_object_store.Del('bob/bob0')
-    file_system._read_object_store.Del('bob/bob0')
-    file_system._stat_object_store.Del('bob/bob1')
+    file_system._stat_cache.Del('bob/bob0')
+    file_system._read_cache.Del('bob/bob0')
+    file_system._stat_cache.Del('bob/bob1')
     test_fs.IncrementStat();
     futures = (file_system.ReadSingle('bob/bob1'),
                file_system.ReadSingle('bob/bob0'))
@@ -128,8 +128,8 @@ class CachingFileSystemTest(unittest.TestCase):
 
     # Test a more recent parent directory doesn't force a refetch of children.
     file_system = create_empty_caching_fs()
-    file_system._read_object_store.Del('bob/bob0')
-    file_system._read_object_store.Del('bob/bob1')
+    file_system._read_cache.Del('bob/bob0')
+    file_system._read_cache.Del('bob/bob1')
     futures = (file_system.ReadSingle('bob/bob1'),
                file_system.ReadSingle('bob/bob2'),
                file_system.ReadSingle('bob/bob3'))
@@ -150,7 +150,7 @@ class CachingFileSystemTest(unittest.TestCase):
     self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
 
     file_system = create_empty_caching_fs()
-    file_system._stat_object_store.Del('bob/bob0')
+    file_system._stat_cache.Del('bob/bob0')
     future = file_system.ReadSingle('bob/bob0')
     self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
     self.assertEqual('bob/bob0 contents', future.Get())
@@ -251,6 +251,44 @@ class CachingFileSystemTest(unittest.TestCase):
       'bob/bob0': 'bob/bob0 contents',
     }, read_skip_not_found(('bob/bob0', 'bob/bob2')))
 
+  def testWalkCaching(self):
+    test_fs = TestFileSystem({
+      'root': {
+        'file1': 'file1',
+        'file2': 'file2',
+        'dir1': {
+          'dir1_file1': 'dir1_file1',
+          'dir2': {},
+          'dir3': {
+            'dir3_file1': 'dir3_file1',
+            'dir3_file2': 'dir3_file2'
+          }
+        }
+      }
+    })
+    mock_fs = MockFileSystem(test_fs)
+    file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
+    for walkinfo in file_system.Walk(''):
+      pass
+    self.assertTrue(*mock_fs.CheckAndReset(
+        read_resolve_count=5, read_count=5, stat_count=5))
+
+    all_dirs, all_files = [], []
+    for root, dirs, files in file_system.Walk(''):
+      all_dirs.extend(dirs)
+      all_files.extend(files)
+    self.assertEqual(sorted(['root/', 'dir1/', 'dir2/', 'dir3/']),
+                     sorted(all_dirs))
+    self.assertEqual(
+        sorted(['file1', 'file2', 'dir1_file1', 'dir3_file1', 'dir3_file2']),
+        sorted(all_files))
+    # All data should be cached.
+    self.assertTrue(*mock_fs.CheckAndReset())
+
+    # Starting from a different root should still pull cached data.
+    for walkinfo in file_system.Walk('root/dir1/'):
+      pass
+    self.assertTrue(*mock_fs.CheckAndReset())

[not at google - send to devlin, 2014/08/29 20:26:07]

Could you also test with a new CachingFileSystem? Calling
_CreateCachingFileSystem multiple times from the same test will result in the
same object store being returned, so I'd expect a Stat of 1 each time. Or maybe
5 each time. Ideally 1 because Walk could be smart enough to know that the file
system hasn't changed by Stat-ing the root. Maybe StatAsync already has this
optimisation and it's not worth worrying about.

Or you can TODO-ify this. If it still gets a 30% improvement we should submit it
now.

 
 if __name__ == '__main__':
   unittest.main()