Docserver: Override Walk in CachingFileSystem

chrome/common/extensions/docs/server2/caching_file_system.py

Index: chrome/common/extensions/docs/server2/caching_file_system.py
diff --git a/chrome/common/extensions/docs/server2/caching_file_system.py b/chrome/common/extensions/docs/server2/caching_file_system.py
index 016cac3c7579d94524c98a9eab27c177111a174f..07ba72e24583b881150694fce376e266c0d55d8c 100644
--- a/chrome/common/extensions/docs/server2/caching_file_system.py
+++ b/chrome/common/extensions/docs/server2/caching_file_system.py
@@ -7,7 +7,7 @@ import sys
 
 from file_system import FileSystem, StatInfo, FileNotFoundError
 from future import Future
-from path_util import IsDirectory, ToDirectory
+from path_util import AssertIsDirectory, IsDirectory, ToDirectory
 from third_party.json_schema_compiler.memoize import memoize
 
 
@@ -29,6 +29,7 @@ class CachingFileSystem(FileSystem):
     # which starting empty is designed for. Without this optimisation, cron
     # runs are extra slow.
     self._read_object_store = create_object_store('read', start_empty=False)
+    self._walk_cache = create_object_store('walk', start_empty=False)

[not at google - send to devlin, 2014/08/29 19:28:31]

A more consistent name for this would be "_walk_object_store". I do like
"_walk_cache" though, if you feel inspired you could just rename
stat/read_object_store to stat/read_cache.

 
   def Refresh(self):
     return self._file_system.Refresh()
@@ -140,6 +141,28 @@ class CachingFileSystem(FileSystem):
     return self._file_system.Read(set(paths) - set(up_to_date_data.iterkeys()),
                                   skip_not_found=skip_not_found).Then(next)
 
+  def Walk(self, root, depth=-1):
+    '''Overrides FileSystem.Walk() to provide caching functionality.
+    '''
+    def delegate(root):
+      res = self._walk_cache.Get(root).Get()
+      cache_stat = self._stat_object_store.Get(root).Get()
+      root_stat = self.Stat(root)

[not at google - send to devlin, 2014/08/29 19:28:31]

StatAsync.

And you could do these in parallel pretty easily using All(); but should this
follow Read more closely? Like, cache the version inside the walk cache, then
compare that to the version that comes back from StatAsync?

Like not even bother looking up _stat_object_store yourself.

+
+      if not res or (cache_stat and cache_stat.version != root_stat.version):
+        # Wasn't cached, or not up to date.
+        dirs, files = [], []
+        for f in self.ReadSingle(root).Get():
+          if IsDirectory(f):
+            dirs.append(f)
+          else:
+            files.append(f)
+        self._walk_cache.Set(root, (dirs, files))
+      else:
+        dirs, files = res
+      return dirs, files
+    return self._file_system.Walk(root, depth=depth, walk_delegate=delegate)
+
   def GetIdentity(self):
     return self._file_system.GetIdentity()