Docserver: Persist stat cache for versioned file systems

chrome/common/extensions/docs/server2/cron_servlet.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import time
 import traceback
 
 from app_yaml_helper import AppYamlHelper
 from appengine_wrappers import IsDeadlineExceededError, logservice, taskqueue
 from branch_utility import BranchUtility
 from compiled_file_system import CompiledFileSystem
 from custom_logger import CustomLogger
 from data_source_registry import CreateDataSources
 from environment import GetAppVersion
 from gcs_file_system_provider import CloudStorageFileSystemProvider
 from github_file_system_provider import GithubFileSystemProvider
 from host_file_system_provider import HostFileSystemProvider
 from object_store_creator import ObjectStoreCreator
+from refresh_tracker import RefreshTracker
 from render_refresher import RenderRefresher
 from server_instance import ServerInstance
 from servlet import Servlet, Request, Response
 from timer import Timer
 
 
 _log = CustomLogger('cron')
 
 
 class CronServlet(Servlet):
@@ skipping 48 matching lines @@
     # and spin up taskqueue tasks which will refresh any cached data relevant
     # to these assets.
     #
     # TODO(rockot/kalman): At the moment examples are not actually refreshed
     # because they're too slow.
 
     _log.info('starting')
 
     server_instance = self._GetSafeServerInstance()
     master_fs = server_instance.host_file_system_provider.GetMaster()
-    master_commit = master_fs.GetCommitID().Get()
+    if 'commit' in self._request.arguments:
+      master_commit = self._request.arguments['commit']
+    else:
+      master_commit = master_fs.GetCommitID().Get()
 
     # This is the guy that would be responsible for refreshing the cache of
     # examples. Here for posterity, hopefully it will be added to the targets
     # below someday.
     render_refresher = RenderRefresher(server_instance, self._request)
 
+    # Used to register a new refresh cycle keyed on |master_commit|.
+    refresh_tracker = RefreshTracker(server_instance.object_store_creator)
+
     # Get the default taskqueue
     queue = taskqueue.Queue()
 
     # GAE documentation specifies that it's bad to add tasks to a queue
     # within one second of purging. We wait 2 seconds, because we like
     # to go the extra mile.
     queue.purge()
     time.sleep(2)
 
     success = True
     try:
       data_sources = CreateDataSources(server_instance)
       targets = (data_sources.items() +
                  [('content_providers', server_instance.content_providers),
                   ('platform_bundle', server_instance.platform_bundle)])
       title = 'initializing %s parallel targets' % len(targets)
       _log.info(title)
       timer = Timer()
+      tasks = []
       for name, target in targets:
         refresh_paths = target.GetRefreshPaths()
-        for path in refresh_paths:
-          queue.add(taskqueue.Task(url='/_refresh/%s/%s' % (name, path),
-                                   params={'commit': master_commit}))
+        tasks += [('%s/%s' % (name, path)).strip('/') for path in refresh_paths]
+
+      # Start a new refresh cycle. In order to detect the completion of a full
+      # cache refresh, the RefreshServlet (which handles individual refresh
+      # tasks) will mark each task complete and check the set of completed tasks
+      # against the set registered here.
+      refresh_tracker.StartRefresh(master_commit, tasks).Get()
+      for task in tasks:
+        queue.add(taskqueue.Task(url='/_refresh/%s' % task,
+                                 params={'commit': master_commit}))
+
       _log.info('%s took %s' % (title, timer.Stop().FormatElapsed()))
     except:
       # This should never actually happen (each cron step does its own
       # conservative error checking), so re-raise no matter what it is.
       _log.error('uncaught error: %s' % traceback.format_exc())
       success = False
       raise
     finally:
       _log.info('finished (%s)', 'success' if success else 'FAILED')
       return (Response.Ok('Success') if success else
@@ skipping 52 matching lines @@
     github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
         object_store_creator)
     gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider(
         object_store_creator)
     return ServerInstance(object_store_creator,
                           CompiledFileSystem.Factory(object_store_creator),
                           branch_utility,
                           host_file_system_provider,
                           github_file_system_provider,
                           gcs_file_system_provider)