Docserver move cron jobs to a backend instance

chrome/common/extensions/docs/server2/cron_servlet.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import time
 import traceback
 
 from app_yaml_helper import AppYamlHelper
 from appengine_wrappers import (
-    GetAppVersion, DeadlineExceededError, IsDevServer, logservice)

[方觉(Fang Jue), 2013/08/14 01:47:42]

DeadlineExceededError can still happen even if it's a backend and there're more
than one type of DeadlineExceededError. See
https://codereview.chromium.org/22457003/diff/22001/chrome/common/extensions/....

+    GetAppVersion, IsDevServer, logservice)
 from branch_utility import BranchUtility
 from compiled_file_system import CompiledFileSystem
 from empty_dir_file_system import EmptyDirFileSystem
 from file_system_util import CreateURLsFromPaths
 from github_file_system import GithubFileSystem
 from host_file_system_creator import HostFileSystemCreator
 from object_store_creator import ObjectStoreCreator
 from render_servlet import RenderServlet
 from server_instance import ServerInstance
 from servlet import Servlet, Request, Response
@@ skipping 25 matching lines @@
     def CreateAppSamplesFileSystem(self, object_store_creator):
       # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's
       # not supported yet (see comment there).
       return (EmptyDirFileSystem() if IsDevServer() else
               GithubFileSystem.Create(object_store_creator))
 
     def GetAppVersion(self):
       return GetAppVersion()
 
   def Get(self):
-    # Crons often time out, and when they do *and* then eventually try to
-    # flush logs they die. Turn off autoflush and manually do so at the end.
-    logservice.AUTOFLUSH_ENABLED = False
-    try:
-      return self._GetImpl()
-    finally:
-      logservice.flush()
-
-  def _GetImpl(self):
     # Cron strategy:
     #
     # Find all public template files and static files, and render them. Most of
     # the time these won't have changed since the last cron run, so it's a
     # little wasteful, but hopefully rendering is really fast (if it isn't we
     # have a problem).
     logging.info('cron: starting')
 
     # This is returned every time RenderServlet wants to create a new
     # ServerInstance.
     server_instance = self._GetSafeServerInstance()
 
     def get_via_render_servlet(path):
       request = Request(path, self._request.host, self._request.headers)
       delegate = _SingletonRenderServletDelegate(server_instance)
       return RenderServlet(request, delegate).Get()
 
     def run_cron_for_dir(d, path_prefix=''):
       success = True
       start_time = time.time()
       files = dict(
           CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix))
       logging.info('cron: rendering %s files from %s...' % (len(files), d))
       try:
         for i, path in enumerate(files):
-          error = None
-          try:
-            response = get_via_render_servlet(path)
-            if response.status != 200:
-              error = 'Got %s response' % response.status
-          except DeadlineExceededError:
-            logging.error(
-                'cron: deadline exceeded rendering %s (%s of %s): %s' % (
-                    path, i + 1, len(files), traceback.format_exc()))
-            raise
-          except error:
-            pass
-          if error:
-            logging.error('cron: error rendering %s: %s' % (path, error))
+          response = get_via_render_servlet(path)
+          if response.status != 200:
+            logging.error('cron: error rendering %s: %s' % (
+                path, 'Got %s response' % response.status))
             success = False
       finally:
         logging.info('cron: rendering %s files from %s took %s seconds' % (
             len(files), d, time.time() - start_time))
       return success
 
     success = True
-    try:
-      # Render all of the publicly accessible files.
-      cron_runs = [
-        # Note: rendering the public templates will pull in all of the private
-        # templates.
-        (svn_constants.PUBLIC_TEMPLATE_PATH, ''),
-        # Note: rendering the public templates will have pulled in the .js
-        # and manifest.json files (for listing examples on the API reference
-        # pages), but there are still images, CSS, etc.
-        (svn_constants.STATIC_PATH, 'static/'),
-      ]
-      if not IsDevServer():
-        cron_runs.append(
-            (svn_constants.EXAMPLES_PATH, 'extensions/examples/'))
 
-      # Note: don't try to short circuit any of this stuff. We want to run
-      # the cron for all the directories regardless of intermediate
-      # failures.
-      for path, path_prefix in cron_runs:
-        success = run_cron_for_dir(path, path_prefix=path_prefix) and success
+    # Render all of the publicly accessible files.
+    cron_runs = [
+      # Note: rendering the public templates will pull in all of the private
+      # templates.
+      (svn_constants.PUBLIC_TEMPLATE_PATH, ''),
+      # Note: rendering the public templates will have pulled in the .js
+      # and manifest.json files (for listing examples on the API reference
+      # pages), but there are still images, CSS, etc.
+      (svn_constants.STATIC_PATH, 'static/'),
+    ]
+    if not IsDevServer():
+      cron_runs.append(
+          (svn_constants.EXAMPLES_PATH, 'extensions/examples/'))
 
-      # TODO(kalman): Generic way for classes to request cron access. The next
-      # two special cases are ugly. It would potentially greatly speed up cron
-      # runs, too.
+    # Note: don't try to short circuit any of this stuff. We want to run
+    # the cron for all the directories regardless of intermediate
+    # failures.
+    for path, path_prefix in cron_runs:
+      success = run_cron_for_dir(path, path_prefix=path_prefix) and success
 
-      # Extension examples have zip files too. Well, so do apps, but the app
-      # file system doesn't get the Offline treatment so they don't need cron.
-      if not IsDevServer():
-        manifest_json = 'manifest.json'
-        example_zips = []
-        for root, _, files in server_instance.host_file_system.Walk(
-            svn_constants.EXAMPLES_PATH):
-          example_zips.extend(
-              root + '.zip' for name in files if name == manifest_json)
-        logging.info('cron: rendering %s example zips...' % len(example_zips))
-        start_time = time.time()
-        try:
-          success = success and all(
-              get_via_render_servlet('extensions/examples/%s' % z).status == 200
-              for z in example_zips)
-        finally:
-          logging.info('cron: rendering %s example zips took %s seconds' % (
-              len(example_zips), time.time() - start_time))
+    # TODO(kalman): Generic way for classes to request cron access. The next
+    # two special cases are ugly. It would potentially greatly speed up cron
+    # runs, too.
 
-    except DeadlineExceededError:
-      success = False
+    # Extension examples have zip files too. Well, so do apps, but the app
+    # file system doesn't get the Offline treatment so they don't need cron.
+    if not IsDevServer():
+      manifest_json = 'manifest.json'
+      example_zips = []
+      for root, _, files in server_instance.host_file_system.Walk(
+          svn_constants.EXAMPLES_PATH):
+        example_zips.extend(
+            root + '.zip' for name in files if name == manifest_json)
+      logging.info('cron: rendering %s example zips...' % len(example_zips))
+      start_time = time.time()
+      try:
+        success = success and all(
+            get_via_render_servlet('extensions/examples/%s' % z).status == 200
+            for z in example_zips)
+      finally:
+        logging.info('cron: rendering %s example zips took %s seconds' % (
+            len(example_zips), time.time() - start_time))
 
     logging.info('cron: running Redirector cron...')
     server_instance.redirector.Cron()
 
     logging.info('cron: finished (%s)' % ('success' if success else 'failure',))
 
     return (Response.Ok('Success') if success else
             Response.InternalError('Failure'))
 
   def _GetSafeServerInstance(self):
@@ skipping 36 matching lines @@
     compiled_host_fs_factory = CompiledFileSystem.Factory(
         host_file_system,
         object_store_creator)
     return ServerInstance(object_store_creator,
                           host_file_system,
                           app_samples_file_system,
                           '',
                           compiled_host_fs_factory,
                           branch_utility,
                           host_file_system_creator)