Docserver: Persist stat cache for versioned file systems

chrome/common/extensions/docs/server2/refresh_servlet.py

Index: chrome/common/extensions/docs/server2/refresh_servlet.py
diff --git a/chrome/common/extensions/docs/server2/refresh_servlet.py b/chrome/common/extensions/docs/server2/refresh_servlet.py
index eb8c0642a7a67ccf66f876cd8f9d379da6115f47..b91c29ab83f9e661b320e95d4f354305928c7974 100644
--- a/chrome/common/extensions/docs/server2/refresh_servlet.py
+++ b/chrome/common/extensions/docs/server2/refresh_servlet.py
@@ -5,8 +5,9 @@
 import traceback
 
 from app_yaml_helper import AppYamlHelper
-from appengine_wrappers import IsDeadlineExceededError, logservice
+from appengine_wrappers import IsDeadlineExceededError, logservice, taskqueue
 from branch_utility import BranchUtility
+from commit_tracker import CommitTracker
 from compiled_file_system import CompiledFileSystem
 from custom_logger import CustomLogger
 from data_source_registry import CreateDataSource
@@ -17,12 +18,12 @@ from gcs_file_system_provider import CloudStorageFileSystemProvider
 from github_file_system_provider import GithubFileSystemProvider
 from host_file_system_provider import HostFileSystemProvider
 from object_store_creator import ObjectStoreCreator
+from refresh_tracker import RefreshTracker
 from server_instance import ServerInstance
 from servlet import Servlet, Request, Response
 from timer import Timer, TimerClosure
 
 
-
 _log = CustomLogger('refresh')
 
 
@@ -86,15 +87,32 @@ class RefreshServlet(Servlet):
       commit = None
 
     server_instance = self._CreateServerInstance(commit)
+    commit_tracker = CommitTracker(server_instance.object_store_creator)
+    refresh_tracker = RefreshTracker(server_instance.object_store_creator)
+
     success = True
     try:
       if source_name == 'platform_bundle':
         data_source = server_instance.platform_bundle
       elif source_name == 'content_providers':
         data_source = server_instance.content_providers
+      elif source_name == 'instance_master':
+        data_source = _InstanceMasterCommitUpdater(commit)

[not at google - send to devlin, 2014/10/24 00:04:51]

I don't see _InstanceMasterCommitUpdater anywhere?

[Ken Rockot(use gerrit already), 2014/10/24 21:26:54]

Oops. Yeah this is old dead code. Removed.

       else:
         data_source = CreateDataSource(source_name, server_instance)
 
+      # If the data source defines subtasks and none was provided, do the nice
+      # thing and queue up the defined subtasks instead of just failing.
+      if source_path is None:
+        refresh_paths = data_source.GetRefreshPaths()
+        if len(refresh_paths) > 1:
+          queue = taskqueue.Queue()
+          tasks = ['/_refresh/%s/%s' % (source_name, path)
+                   for path in refresh_paths]
+          for task in tasks:
+            queue.add(taskqueue.Task(url=task, params={'commit': commit}))
+          return

[not at google - send to devlin, 2014/10/24 00:04:51]

What happens if source_path is None but there are no refresh paths? It falls
through, and... it crashes?

[Ken Rockot(use gerrit already), 2014/10/24 21:26:54]

This works just fine (when source_path is None it means the top-level task is
the only one for that DataSource; e.g. /_refresh/api_list just does a refresh of
APIListDataSource with no path, where as the apis data source requires
subpaths).

It's confusing though and really only useful for testing (so I could do
something like /_refresh/apis to manually verify all API refreshes without doing
a full cron or individually running _refresh/apis/apps/usb,
_refresh/apis/extensions/tabs, etc).

I don't think it should stay, so I killed it.

+
       class_name = data_source.__class__.__name__
       refresh_future = data_source.Refresh(source_path)
       assert isinstance(refresh_future, Future), (
@@ -102,6 +120,14 @@ class RefreshServlet(Servlet):
       timer = Timer()
       try:
         refresh_future.Get()
+
+        def resolve(refresh_complete):
+          if refresh_complete:
+            return commit_tracker.Set('master', commit)
+          return None
+
+        refresh_tracker.MarkTaskComplete(commit, path).Get()

[not at google - send to devlin, 2014/10/24 00:04:51]

Use Then() not Get()?

You could even go so far as:

(refresh_tracker.MarkTaskComplete(commit, path)
 .Then(lambda: refresh_tracker.GetRefreshComplete(commit))
 .Then(lambda completed:
         commit_tracker.Set('master', commit) if completed else None)
 .Get())

which is pretty fun.

[Ken Rockot(use gerrit already), 2014/10/24 21:26:54]

Neat. I'm all for chaining futures of you're down with that. Some folks are
queasy about anonymous functions. :D

Done.

[not at google - send to devlin, 2014/10/24 23:36:07]

I think the Python creators are against anonymous functions.

+        refresh_tracker.GetRefreshComplete(commit).Then(resolve).Get()

[not at google - send to devlin, 2014/10/24 00:04:52]

You should mention how this relates to cron servlet.

[Ken Rockot(use gerrit already), 2014/10/24 21:26:54]

Done.

       except Exception as e:
         _log.error('%s: error %s' % (class_name, traceback.format_exc()))
         success = False