Docserver: Pull knowledge of host file systems into a single

chrome/common/extensions/docs/server2/cron_servlet.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import time
 import traceback
 
 from app_yaml_helper import AppYamlHelper
 from appengine_wrappers import (
     GetAppVersion, IsDeadlineExceededError, IsDevServer, logservice)
 from branch_utility import BranchUtility
 from compiled_file_system import CompiledFileSystem
 from data_source_registry import CreateDataSources
 from empty_dir_file_system import EmptyDirFileSystem
 from file_system_util import CreateURLsFromPaths
 from github_file_system import GithubFileSystem
-from host_file_system_creator import HostFileSystemCreator
+from host_file_system_provider import HostFileSystemProvider
 from object_store_creator import ObjectStoreCreator
 from render_servlet import RenderServlet
 from server_instance import ServerInstance
 from servlet import Servlet, Request, Response
 import svn_constants
 
 class _SingletonRenderServletDelegate(RenderServlet.Delegate):
   def __init__(self, server_instance):
     self._server_instance = server_instance
 
@@ skipping 54 matching lines @@
   def __init__(self, request, delegate_for_test=None):
     Servlet.__init__(self, request)
     self._delegate = delegate_for_test or CronServlet.Delegate()
 
   class Delegate(object):
     '''CronServlet's runtime dependencies. Override for testing.
     '''
     def CreateBranchUtility(self, object_store_creator):
       return BranchUtility.Create(object_store_creator)
 
-    def CreateHostFileSystemCreator(self, object_store_creator):
-      return HostFileSystemCreator(object_store_creator)
+    def CreateHostFileSystemProvider(self,
+                                     object_store_creator,
+                                     max_trunk_revision=None):
+      return HostFileSystemProvider(object_store_creator,
+                                    max_trunk_revision=max_trunk_revision)
 
     def CreateAppSamplesFileSystem(self, object_store_creator):
       # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's
       # not supported yet (see comment there).
       return (EmptyDirFileSystem() if IsDevServer() else
               GithubFileSystem.Create(object_store_creator))
 
     def GetAppVersion(self):
       return GetAppVersion()
 
@@ skipping 21 matching lines @@
     # little wasteful, but hopefully rendering is really fast (if it isn't we
     # have a problem).
     _cronlog.info('starting')
 
     # This is returned every time RenderServlet wants to create a new
     # ServerInstance.
     #
     # TODO(kalman): IMPORTANT. This sometimes throws an exception, breaking
     # everything. Need retry logic at the fetcher level.
     server_instance = self._GetSafeServerInstance()
+    trunk_fs = server_instance.host_file_system_provider.GetTrunk()
 
     def render(path):
       request = Request(path, self._request.host, self._request.headers)
       delegate = _SingletonRenderServletDelegate(server_instance)
       return RenderServlet(request, delegate).Get()
 
     def request_files_in_dir(path, prefix=''):
       '''Requests every file found under |path| in this host file system, with
       a request prefix of |prefix|.
       '''
-      files = [name for name, _ in
-          CreateURLsFromPaths(server_instance.host_file_system, path, prefix)]
+      files = [name for name, _ in CreateURLsFromPaths(trunk_fs, path, prefix)]
       return _RequestEachItem(path, files, render)
 
     results = []
 
     try:
       # Rendering the public templates will also pull in all of the private
       # templates.
       results.append(request_files_in_dir(svn_constants.PUBLIC_TEMPLATE_PATH))
 
       # Rendering the public templates will have pulled in the .js and
       # manifest.json files (for listing examples on the API reference pages),
       # but there are still images, CSS, etc.
       results.append(request_files_in_dir(svn_constants.STATIC_PATH,
                                           prefix='static/'))
 
       # Samples are too expensive to run on the dev server, where there is no
       # parallel fetch.
       if not IsDevServer():
         # Fetch each individual sample file.
         results.append(request_files_in_dir(svn_constants.EXAMPLES_PATH,
                                             prefix='extensions/examples/'))
 
         # Fetch the zip file of each example (contains all the individual
         # files).
         example_zips = []
-        for root, _, files in server_instance.host_file_system.Walk(
-            svn_constants.EXAMPLES_PATH):
+        for root, _, files in trunk_fs.Walk(svn_constants.EXAMPLES_PATH):
           example_zips.extend(
               root + '.zip' for name in files if name == 'manifest.json')
         results.append(_RequestEachItem(
             'example zips',
             example_zips,
             lambda path: render('extensions/examples/' + path)))
 
       def run_cron(data_source):
         title = data_source.__class__.__name__
         _cronlog.info('%s: starting' % title)
@@ skipping 35 matching lines @@
     # IMPORTANT: Get a ServerInstance pinned to the most recent revision, not
     # HEAD. These cron jobs take a while and run very frequently such that
     # there is usually one running at any given time, and eventually a file
     # that we're dealing with will change underneath it, putting the server in
     # an undefined state.
     server_instance_near_head = self._CreateServerInstance(
         self._GetMostRecentRevision())
 
     app_yaml_handler = AppYamlHelper(
         svn_constants.APP_YAML_PATH,
-        server_instance_near_head.host_file_system,
         server_instance_near_head.object_store_creator,
-        server_instance_near_head.host_file_system_creator)
+        server_instance_near_head.host_file_system_provider)
 
     if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
       return server_instance_near_head
 
     # The version in app.yaml is greater than the currently running app's.
     # The safe version is the one before it changed.
     safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
         delegate.GetAppVersion()) - 1
 
     _cronlog.info('app version %s is out of date, safe is %s',
         delegate.GetAppVersion(), safe_revision)
 
     return self._CreateServerInstance(safe_revision)
 
   def _GetMostRecentRevision(self):
     '''Gets the revision of the most recent patch submitted to the host file
     system. This is similar to HEAD but it's a concrete revision so won't
     change as the cron runs.
     '''
-    return self._CreateServerInstance(None).host_file_system.Stat('/').version
+    head_fs = (
+        self._CreateServerInstance(None).host_file_system_provider.GetTrunk())
+    return head_fs.Stat('/').version
 
   def _CreateServerInstance(self, revision):
     '''Creates a ServerInstance pinned to |revision|, or HEAD if None.
     NOTE: If passed None it's likely that during the cron run patches will be
     submitted at HEAD, which may change data underneath the cron run.
     '''
     object_store_creator = ObjectStoreCreator(start_empty=True)
     branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
-    host_file_system_creator = self._delegate.CreateHostFileSystemCreator(
-        object_store_creator)
-    host_file_system = host_file_system_creator.Create(revision=revision)
+    host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
+        object_store_creator, max_trunk_revision=revision)
     app_samples_file_system = self._delegate.CreateAppSamplesFileSystem(
         object_store_creator)
     compiled_host_fs_factory = CompiledFileSystem.Factory(
-        host_file_system,
+        host_file_system_provider.GetTrunk(),
         object_store_creator)
     return ServerInstance(object_store_creator,
-                          host_file_system,
                           app_samples_file_system,
                           compiled_host_fs_factory,
                           branch_utility,
-                          host_file_system_creator)
+                          host_file_system_provider)