Docserver: Persist stat cache for versioned file systems

chrome/common/extensions/docs/server2/refresh_servlet.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import traceback
 
 from app_yaml_helper import AppYamlHelper
-from appengine_wrappers import IsDeadlineExceededError, logservice
+from appengine_wrappers import IsDeadlineExceededError, logservice, taskqueue
 from branch_utility import BranchUtility
+from commit_tracker import CommitTracker
 from compiled_file_system import CompiledFileSystem
 from custom_logger import CustomLogger
 from data_source_registry import CreateDataSource
 from environment import GetAppVersion
 from file_system import IsFileSystemThrottledError
 from future import Future
 from gcs_file_system_provider import CloudStorageFileSystemProvider
 from github_file_system_provider import GithubFileSystemProvider
 from host_file_system_provider import HostFileSystemProvider
 from object_store_creator import ObjectStoreCreator
+from refresh_tracker import RefreshTracker
 from server_instance import ServerInstance
 from servlet import Servlet, Request, Response
 from timer import Timer, TimerClosure
 
 
+_log = CustomLogger('refresh')
 
-_log = CustomLogger('refresh')
+
+def _UpdateMasterCommit(object_store_creator, commit):
+  commit_tracker = CommitTracker(object_store_creator)
+  return commit_tracker.Set('master', commit)
 
 
 class RefreshServlet(Servlet):
   '''Servlet which refreshes a single data source.
   '''
   def __init__(self, request, delegate_for_test=None):
     Servlet.__init__(self, request)
     self._delegate = delegate_for_test or RefreshServlet.Delegate()
 
   class Delegate(object):
@@ skipping 42 matching lines @@
         (source_name, '' if source_path is None else '[%s]' % source_path))
 
     if 'commit' in self._request.arguments:
       commit = self._request.arguments['commit']
     else:
       _log.warning('No commit given; refreshing from master. '
                    'This is probably NOT what you want.')
       commit = None
 
     server_instance = self._CreateServerInstance(commit)
+    refresh_tracker = RefreshTracker(server_instance.object_store_creator)
+
     success = True
+    success_message = 'Success'
     try:
       if source_name == 'platform_bundle':
         data_source = server_instance.platform_bundle
       elif source_name == 'content_providers':
         data_source = server_instance.content_providers
+      elif source_name == 'instance_master':
+        data_source = _InstanceMasterCommitUpdater(commit)
       else:
         data_source = CreateDataSource(source_name, server_instance)
 
+      # If the data source defines subtasks and none was provided, do the nice
+      # thing and queue up the defined subtasks instead of just failing.
+      if source_path is None:
+        refresh_paths = data_source.GetRefreshPaths()
+        if len(refresh_paths) > 1:
+          queue = taskqueue.Queue()
+          tasks = ['/_refresh/%s/%s' % (source_name, path)
+                   for path in refresh_paths]
+          for task in tasks:
+            queue.add(taskqueue.Task(url=task, params={'commit': commit}))
+          success_message = 'Enqueued tasks: %s' % tasks
+          # The finally clause really handles the return.
+          return
+
       class_name = data_source.__class__.__name__
       refresh_future = data_source.Refresh(source_path)
       assert isinstance(refresh_future, Future), (
           '%s.Refresh() did not return a Future' % class_name)
       timer = Timer()
       try:
         refresh_future.Get()
+
+        def resolve(refresh_complete):
+          if refresh_complete:
+            return _UpdateMasterCommit(
+                server_instance.object_store_creator, commit)
+          return None
+        refresh_tracker.MarkTaskComplete(commit, '_refresh/%s/%s' % (
+            source_name, path)).Get()
+        refresh_tracker.GetRefreshComplete(commit).Then(resolve).Get()
       except Exception as e:
         _log.error('%s: error %s' % (class_name, traceback.format_exc()))
         success = False
         if IsFileSystemThrottledError(e):
           return Response.ThrottledError('Throttled')
         raise
       finally:
         _log.info('Refreshing %s took %s' %
             (class_name, timer.Stop().FormatElapsed()))
 
     except:
       success = False
       # This should never actually happen.
       _log.error('uncaught error: %s' % traceback.format_exc())
       raise
     finally:
       _log.info('finished (%s)', 'success' if success else 'FAILED')
-      return (Response.Ok('Success') if success else
+      return (Response.Ok(success_message) if success else
               Response.InternalError('Failure'))
 
   def _CreateServerInstance(self, commit):
     '''Creates a ServerInstance pinned to |commit|, or HEAD if None.
     NOTE: If passed None it's likely that during the cron run patches will be
     submitted at HEAD, which may change data underneath the cron run.
     '''
     object_store_creator = ObjectStoreCreator(start_empty=True)
     branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
     host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
         object_store_creator, pinned_commit=commit)
     github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
         object_store_creator)
     gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider(
         object_store_creator)
     return ServerInstance(object_store_creator,
                           CompiledFileSystem.Factory(object_store_creator),
                           branch_utility,
                           host_file_system_provider,
                           github_file_system_provider,
                           gcs_file_system_provider)