Docserver: Gitiles auth and cron refactoring.

chrome/common/extensions/docs/server2/refresh_servlet.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import traceback
+
+from app_yaml_helper import AppYamlHelper
+from appengine_wrappers import IsDeadlineExceededError, logservice
+from branch_utility import BranchUtility
+from compiled_file_system import CompiledFileSystem
+from custom_logger import CustomLogger
+from data_source_registry import CreateDataSource
+from environment import GetAppVersion
+from file_system import IsFileSystemThrottledError
+from future import Future
+from gcs_file_system_provider import CloudStorageFileSystemProvider
+from github_file_system_provider import GithubFileSystemProvider
+from host_file_system_provider import HostFileSystemProvider
+from object_store_creator import ObjectStoreCreator
+from server_instance import ServerInstance
+from servlet import Servlet, Request, Response
+from timer import Timer, TimerClosure
+
+
+
+_log = CustomLogger('refresh')
+
+
+class RefreshServlet(Servlet):
+  '''Servlet which refreshes a single data source.
+  '''
+  def __init__(self, request, delegate_for_test=None):
+    Servlet.__init__(self, request)
+    self._delegate = delegate_for_test or RefreshServlet.Delegate()
+
+  class Delegate(object):
+    '''RefreshServlet's runtime dependencies. Override for testing.
+    '''
+    def CreateBranchUtility(self, object_store_creator):
+      return BranchUtility.Create(object_store_creator)
+
+    def CreateHostFileSystemProvider(self,
+                                     object_store_creator,
+                                     pinned_commit=None):
+      return HostFileSystemProvider(object_store_creator,
+                                    pinned_commit=pinned_commit)
+
+    def CreateGithubFileSystemProvider(self, object_store_creator):
+      return GithubFileSystemProvider(object_store_creator)
+
+    def CreateGCSFileSystemProvider(self, object_store_creator):
+      return CloudStorageFileSystemProvider(object_store_creator)
+
+    def GetAppVersion(self):
+      return GetAppVersion()
+
+  def Get(self):
+    # Manually flush logs at the end of the run. However, sometimes
+    # even that isn't enough, which is why in this file we use the
+    # custom logger and make it flush the log every time its used.
+    logservice.AUTOFLUSH_ENABLED = False
+    try:
+      return self._GetImpl()
+    except BaseException:
+      _log.error('Caught top-level exception! %s', traceback.format_exc())
+    finally:
+      logservice.flush()
+
+  def _GetImpl(self):
+    path = self._request.path.strip('/')
+    parts = self._request.path.split('/', 1)
+    source_name = parts[0]
+    if len(parts) == 2:
+      source_path = parts[1]
+    else:
+      source_path = None
+
+    _log.info('starting refresh of %s DataSource %s' %
+        (source_name, '' if source_path is None else '[%s]' % source_path))
+
+    if 'commit' in self._request.arguments:
+      commit = self._request.arguments['commit']
+    else:
+      _log.warning('No commit given; refreshing from master. '
+                   'This is probably NOT what you want.')
+      commit = None
+
+    server_instance = self._CreateServerInstance(commit)
+    success = True
+    try:
+      if source_name == 'platform_bundle':
+        data_source = server_instance.platform_bundle
+      elif source_name == 'content_providers':
+        data_source = server_instance.content_providers
+      else:
+        data_source = CreateDataSource(source_name, server_instance)
+
+      class_name = data_source.__class__.__name__
+      refresh_future = data_source.Refresh(source_path)
+      assert isinstance(refresh_future, Future), (
+          '%s.Refresh() did not return a Future' % class_name)
+      timer = Timer()
+      try:
+        refresh_future.Get()
+      except Exception as e:
+        _log.error('%s: error %s' % (class_name, traceback.format_exc()))
+        success = False
+        if IsFileSystemThrottledError(e):
+          return Response.ThrottledError('Throttled')
+        raise
+      finally:
+        _log.info('Refreshing %s took %s' %
+            (class_name, timer.Stop().FormatElapsed()))
+
+    except:
+      success = False
+      # This should never actually happen.
+      _log.error('uncaught error: %s' % traceback.format_exc())
+      raise
+    finally:
+      _log.info('finished (%s)', 'success' if success else 'FAILED')
+      return (Response.Ok('Success') if success else
+              Response.InternalError('Failure'))
+
+  def _CreateServerInstance(self, commit):
+    '''Creates a ServerInstance pinned to |commit|, or HEAD if None.
+    NOTE: If passed None it's likely that during the cron run patches will be
+    submitted at HEAD, which may change data underneath the cron run.
+    '''
+    object_store_creator = ObjectStoreCreator(start_empty=True)
+    branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
+    host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
+        object_store_creator, pinned_commit=commit)
+    github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
+        object_store_creator)
+    gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider(
+        object_store_creator)
+    return ServerInstance(object_store_creator,
+                          CompiledFileSystem.Factory(object_store_creator),
+                          branch_utility,
+                          host_file_system_provider,
+                          github_file_system_provider,
+                          gcs_file_system_provider)