Docserver: Persist stat cache for versioned file systems

chrome/common/extensions/docs/server2/cron_servlet.py

Index: chrome/common/extensions/docs/server2/cron_servlet.py
diff --git a/chrome/common/extensions/docs/server2/cron_servlet.py b/chrome/common/extensions/docs/server2/cron_servlet.py
index 1c9bb93f261ed81770b2d59ec23bf9a482bc94a0..534c1c788c9baa2bcb6c9e423806fd7ad2509a68 100644
--- a/chrome/common/extensions/docs/server2/cron_servlet.py
+++ b/chrome/common/extensions/docs/server2/cron_servlet.py
@@ -16,6 +16,7 @@ from gcs_file_system_provider import CloudStorageFileSystemProvider
 from github_file_system_provider import GithubFileSystemProvider
 from host_file_system_provider import HostFileSystemProvider
 from object_store_creator import ObjectStoreCreator
+from refresh_tracker import RefreshTracker
 from render_refresher import RenderRefresher
 from server_instance import ServerInstance
 from servlet import Servlet, Request, Response
@@ -84,13 +85,19 @@ class CronServlet(Servlet):
 
     server_instance = self._GetSafeServerInstance()
     master_fs = server_instance.host_file_system_provider.GetMaster()
-    master_commit = master_fs.GetCommitID().Get()
+    if 'commit' in self._request.arguments:
+      master_commit = self._request.arguments['commit']
+    else:
+      master_commit = master_fs.GetCommitID().Get()
 
     # This is the guy that would be responsible for refreshing the cache of
     # examples. Here for posterity, hopefully it will be added to the targets
     # below someday.
     render_refresher = RenderRefresher(server_instance, self._request)
 
+    # Used to register a new refresh cycle keyed on |master_commit|.
+    refresh_tracker = RefreshTracker(server_instance.object_store_creator)
+
     # Get the default taskqueue
     queue = taskqueue.Queue()
 
@@ -109,11 +116,17 @@ class CronServlet(Servlet):
       title = 'initializing %s parallel targets' % len(targets)
       _log.info(title)
       timer = Timer()
+      tasks = []
       for name, target in targets:
         refresh_paths = target.GetRefreshPaths()
         for path in refresh_paths:
-          queue.add(taskqueue.Task(url='/_refresh/%s/%s' % (name, path),
-                                   params={'commit': master_commit}))
+          tasks.append(('%s/%s' % (name, path)).strip('/'))

[not at google - send to devlin, 2014/10/24 00:04:51]

nit: list comp? tasks += [('%s/%s' ...)]

[Ken Rockot(use gerrit already), 2014/10/24 21:26:53]

Done.

+
+      refresh_tracker.RegisterRefresh(master_commit, tasks).Get()

[not at google - send to devlin, 2014/10/24 00:04:51]

Maybe this should actually be StartRefresh or something? And you should comment
that the other side of this is the RefreshTracker that gets constructed in
refresh_servlet.

[Ken Rockot(use gerrit already), 2014/10/24 21:26:53]

Done.

+      for task in tasks:
+        queue.add(taskqueue.Task(url='_refresh/%s' % task,
+                                 params={'commit': master_commit}))
+
       _log.info('%s took %s' % (title, timer.Stop().FormatElapsed()))
     except:
       # This should never actually happen (each cron step does its own