| OLD | NEW |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import logging | 5 import logging |
| 6 import time | 6 import time |
| 7 import traceback | 7 import traceback |
| 8 | 8 |
| 9 from app_yaml_helper import AppYamlHelper | 9 from app_yaml_helper import AppYamlHelper |
| 10 from appengine_wrappers import ( | 10 from appengine_wrappers import ( |
| 11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice) | 11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice) |
| 12 from branch_utility import BranchUtility | |
| 13 from caching_file_system import CachingFileSystem | 12 from caching_file_system import CachingFileSystem |
| 14 from compiled_file_system import CompiledFileSystem | 13 from compiled_file_system import CompiledFileSystem |
| 15 from empty_dir_file_system import EmptyDirFileSystem | 14 from empty_dir_file_system import EmptyDirFileSystem |
| 16 from github_file_system import GithubFileSystem | 15 from github_file_system import GithubFileSystem |
| 17 from object_store_creator import ObjectStoreCreator | 16 from object_store_creator import ObjectStoreCreator |
| 18 from render_servlet import RenderServlet | 17 from render_servlet import RenderServlet |
| 19 from server_instance import ServerInstance | 18 from server_instance import ServerInstance |
| 20 from servlet import Servlet, Request, Response | 19 from servlet import Servlet, Request, Response |
| 21 from subversion_file_system import SubversionFileSystem | 20 from subversion_file_system import SubversionFileSystem |
| 22 import svn_constants | 21 import svn_constants |
| 23 from third_party.json_schema_compiler.memoize import memoize | 22 from third_party.json_schema_compiler.memoize import memoize |
| 24 | 23 |
| 25 class _SingletonRenderServletDelegate(RenderServlet.Delegate): | 24 class _SingletonRenderServletDelegate(RenderServlet.Delegate): |
| 26 def __init__(self, server_instance): | 25 def __init__(self, server_instance): |
| 27 self._server_instance = server_instance | 26 self._server_instance = server_instance |
| 28 | 27 |
| 29 def CreateServerInstanceForChannel(self, channel): | 28 def CreateServerInstance(self): |
| 30 return self._server_instance | 29 return self._server_instance |
| 31 | 30 |
| 32 class CronServlet(Servlet): | 31 class CronServlet(Servlet): |
| 33 '''Servlet which runs a cron job. | 32 '''Servlet which runs a cron job. |
| 34 ''' | 33 ''' |
| 35 def __init__(self, request, delegate_for_test=None): | 34 def __init__(self, request, delegate_for_test=None): |
| 36 Servlet.__init__(self, request) | 35 Servlet.__init__(self, request) |
| 37 self._channel = request.path.strip('/') | |
| 38 self._delegate = delegate_for_test or CronServlet.Delegate() | 36 self._delegate = delegate_for_test or CronServlet.Delegate() |
| 39 | 37 |
| 40 class Delegate(object): | 38 class Delegate(object): |
| 41 '''CronServlet's runtime dependencies. Override for testing. | 39 '''CronServlet's runtime dependencies. Override for testing. |
| 42 ''' | 40 ''' |
| 43 def CreateBranchUtility(self, object_store_creator): | 41 def CreateHostFileSystemForRevision(self, revision): |
| 44 return BranchUtility.Create(object_store_creator) | 42 return SubversionFileSystem.Create('trunk', revision=revision) |
| 45 | |
| 46 def CreateHostFileSystemForBranchAndRevision(self, branch, revision): | |
| 47 return SubversionFileSystem.Create(branch, revision=revision) | |
| 48 | 43 |
| 49 def CreateAppSamplesFileSystem(self, object_store_creator): | 44 def CreateAppSamplesFileSystem(self, object_store_creator): |
| 50 # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's | 45 # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's |
| 51 # not supported yet (see comment there). | 46 # not supported yet (see comment there). |
| 52 return (EmptyDirFileSystem() if IsDevServer() else | 47 return (EmptyDirFileSystem() if IsDevServer() else |
| 53 GithubFileSystem.Create(object_store_creator)) | 48 GithubFileSystem.Create(object_store_creator)) |
| 54 | 49 |
| 55 def GetAppVersion(self): | 50 def GetAppVersion(self): |
| 56 return GetAppVersion() | 51 return GetAppVersion() |
| 57 | 52 |
| 58 def Get(self): | 53 def Get(self): |
| 59 # Crons often time out, and when they do *and* then eventually try to | 54 # Crons often time out, and when they do *and* then eventually try to |
| 60 # flush logs they die. Turn off autoflush and manually do so at the end. | 55 # flush logs they die. Turn off autoflush and manually do so at the end. |
| 61 logservice.AUTOFLUSH_ENABLED = False | 56 logservice.AUTOFLUSH_ENABLED = False |
| 62 try: | 57 try: |
| 63 return self._GetImpl() | 58 return self._GetImpl() |
| 64 finally: | 59 finally: |
| 65 logservice.flush() | 60 logservice.flush() |
| 66 | 61 |
| 67 def _GetImpl(self): | 62 def _GetImpl(self): |
| 68 # Cron strategy: | 63 # Cron strategy: |
| 69 # | 64 # |
| 70 # Find all public template files and static files, and render them. Most of | 65 # Find all public template files and static files, and render them. Most of |
| 71 # the time these won't have changed since the last cron run, so it's a | 66 # the time these won't have changed since the last cron run, so it's a |
| 72 # little wasteful, but hopefully rendering is really fast (if it isn't we | 67 # little wasteful, but hopefully rendering is really fast (if it isn't we |
| 73 # have a problem). | 68 # have a problem). |
| 74 channel = self._channel | 69 logging.info('cron: starting') |
| 75 logging.info('cron/%s: starting' % channel) | |
| 76 | 70 |
| 77 # This is returned every time RenderServlet wants to create a new | 71 # This is returned every time RenderServlet wants to create a new |
| 78 # ServerInstance. | 72 # ServerInstance. |
| 79 server_instance = self._GetSafeServerInstance() | 73 server_instance = self._GetSafeServerInstance() |
| 80 | 74 |
| 81 def get_via_render_servlet(path): | 75 def get_via_render_servlet(path): |
| 82 return RenderServlet( | 76 return RenderServlet( |
| 83 Request(path, self._request.host, self._request.headers), | 77 Request(path, self._request.host, self._request.headers), |
| 84 _SingletonRenderServletDelegate(server_instance)).Get() | 78 _SingletonRenderServletDelegate(server_instance)).Get() |
| 85 | 79 |
| 86 def run_cron_for_dir(d, path_prefix=''): | 80 def run_cron_for_dir(d, path_prefix=''): |
| 87 success = True | 81 success = True |
| 88 start_time = time.time() | 82 start_time = time.time() |
| 89 files = [f for f in server_instance.content_cache.GetFromFileListing(d) | 83 files = [f for f in server_instance.content_cache.GetFromFileListing(d) |
| 90 if not f.endswith('/')] | 84 if not f.endswith('/')] |
| 91 logging.info('cron/%s: rendering %s files from %s...' % ( | 85 logging.info('cron: rendering %s files from %s...' % (len(files), d)) |
| 92 channel, len(files), d)) | |
| 93 try: | 86 try: |
| 94 for i, f in enumerate(files): | 87 for i, f in enumerate(files): |
| 95 error = None | 88 error = None |
| 96 path = '%s%s' % (path_prefix, f) | 89 path = '%s%s' % (path_prefix, f) |
| 97 try: | 90 try: |
| 98 response = get_via_render_servlet(path) | 91 response = get_via_render_servlet(path) |
| 99 if response.status != 200: | 92 if response.status != 200: |
| 100 error = 'Got %s response' % response.status | 93 error = 'Got %s response' % response.status |
| 101 except DeadlineExceededError: | 94 except DeadlineExceededError: |
| 102 logging.error( | 95 logging.error( |
| 103 'cron/%s: deadline exceeded rendering %s (%s of %s): %s' % ( | 96 'cron: deadline exceeded rendering %s (%s of %s): %s' % ( |
| 104 channel, path, i + 1, len(files), traceback.format_exc())) | 97 path, i + 1, len(files), traceback.format_exc())) |
| 105 raise | 98 raise |
| 106 except error: | 99 except error: |
| 107 pass | 100 pass |
| 108 if error: | 101 if error: |
| 109 logging.error('cron/%s: error rendering %s: %s' % ( | 102 logging.error('cron: error rendering %s: %s' % (path, error)) |
| 110 channel, path, error)) | |
| 111 success = False | 103 success = False |
| 112 finally: | 104 finally: |
| 113 logging.info('cron/%s: rendering %s files from %s took %s seconds' % ( | 105 logging.info('cron: rendering %s files from %s took %s seconds' % ( |
| 114 channel, len(files), d, time.time() - start_time)) | 106 len(files), d, time.time() - start_time)) |
| 115 return success | 107 return success |
| 116 | 108 |
| 117 success = True | 109 success = True |
| 118 try: | 110 try: |
| 119 # Render all of the publicly accessible files. | 111 # Render all of the publicly accessible files. |
| 120 cron_runs = [ | 112 cron_runs = [ |
| 121 # Note: rendering the public templates will pull in all of the private | 113 # Note: rendering the public templates will pull in all of the private |
| 122 # templates. | 114 # templates. |
| 123 (svn_constants.PUBLIC_TEMPLATE_PATH, ''), | 115 (svn_constants.PUBLIC_TEMPLATE_PATH, ''), |
| 124 # Note: rendering the public templates will have pulled in the .js | 116 # Note: rendering the public templates will have pulled in the .js |
| (...skipping 17 matching lines...) Expand all Loading... |
| 142 | 134 |
| 143 # Extension examples have zip files too. Well, so do apps, but the app | 135 # Extension examples have zip files too. Well, so do apps, but the app |
| 144 # file system doesn't get the Offline treatment so they don't need cron. | 136 # file system doesn't get the Offline treatment so they don't need cron. |
| 145 if not IsDevServer(): | 137 if not IsDevServer(): |
| 146 manifest_json = '/manifest.json' | 138 manifest_json = '/manifest.json' |
| 147 example_zips = [ | 139 example_zips = [ |
| 148 '%s.zip' % filename[:-len(manifest_json)] | 140 '%s.zip' % filename[:-len(manifest_json)] |
| 149 for filename in server_instance.content_cache.GetFromFileListing( | 141 for filename in server_instance.content_cache.GetFromFileListing( |
| 150 svn_constants.EXAMPLES_PATH) | 142 svn_constants.EXAMPLES_PATH) |
| 151 if filename.endswith(manifest_json)] | 143 if filename.endswith(manifest_json)] |
| 152 logging.info('cron/%s: rendering %s example zips...' % ( | 144 logging.info('cron: rendering %s example zips...' % len(example_zips)) |
| 153 channel, len(example_zips))) | |
| 154 start_time = time.time() | 145 start_time = time.time() |
| 155 try: | 146 try: |
| 156 success = success and all( | 147 success = success and all( |
| 157 get_via_render_servlet('extensions/examples/%s' % z).status == 200 | 148 get_via_render_servlet('extensions/examples/%s' % z).status == 200 |
| 158 for z in example_zips) | 149 for z in example_zips) |
| 159 finally: | 150 finally: |
| 160 logging.info('cron/%s: rendering %s example zips took %s seconds' % ( | 151 logging.info('cron: rendering %s example zips took %s seconds' % ( |
| 161 channel, len(example_zips), time.time() - start_time)) | 152 len(example_zips), time.time() - start_time)) |
| 162 | 153 |
| 163 # Also trigger a redirect so that PathCanonicalizer has an opportunity to | 154 # Also trigger a redirect so that PathCanonicalizer has an opportunity to |
| 164 # cache file listings. | 155 # cache file listings. |
| 165 logging.info('cron/%s: triggering a redirect...' % channel) | 156 logging.info('cron: triggering a redirect...') |
| 166 redirect_response = get_via_render_servlet('storage.html') | 157 redirect_response = get_via_render_servlet('storage.html') |
| 167 success = success and redirect_response.status == 302 | 158 success = success and redirect_response.status == 302 |
| 168 except DeadlineExceededError: | 159 except DeadlineExceededError: |
| 169 success = False | 160 success = False |
| 170 | 161 |
| 171 logging.info('cron/%s: finished' % channel) | 162 logging.info('cron: finished (%s)' % 'success' if success else 'failure') |
| 172 | 163 |
| 173 return (Response.Ok('Success') if success else | 164 return (Response.Ok('Success') if success else |
| 174 Response.InternalError('Failure')) | 165 Response.InternalError('Failure')) |
| 175 | 166 |
| 176 def _GetSafeServerInstance(self): | 167 def _GetSafeServerInstance(self): |
| 177 '''Returns a ServerInstance with a host file system at a safe revision, | 168 '''Returns a ServerInstance with a host file system at a safe revision, |
| 178 meaning the last revision that the current running version of the server | 169 meaning the last revision that the current running version of the server |
| 179 existed. | 170 existed. |
| 180 ''' | 171 ''' |
| 181 channel = self._channel | |
| 182 delegate = self._delegate | 172 delegate = self._delegate |
| 183 | 173 |
| 184 server_instance_at_head = self._CreateServerInstance(channel, None) | 174 server_instance_at_head = self._CreateServerInstance(None) |
| 185 | 175 |
| 186 get_branch_for_channel = self._GetBranchForChannel | |
| 187 class AppYamlHelperDelegate(AppYamlHelper.Delegate): | 176 class AppYamlHelperDelegate(AppYamlHelper.Delegate): |
| 188 def GetHostFileSystemForRevision(self, revision): | 177 def GetHostFileSystemForRevision(self, revision): |
| 189 return delegate.CreateHostFileSystemForBranchAndRevision( | 178 return delegate.CreateHostFileSystemForRevision(revision) |
| 190 get_branch_for_channel(channel), | |
| 191 revision) | |
| 192 | 179 |
| 193 app_yaml_handler = AppYamlHelper( | 180 app_yaml_handler = AppYamlHelper( |
| 194 svn_constants.APP_YAML_PATH, | 181 svn_constants.APP_YAML_PATH, |
| 195 server_instance_at_head.host_file_system, | 182 server_instance_at_head.host_file_system, |
| 196 AppYamlHelperDelegate(), | 183 AppYamlHelperDelegate(), |
| 197 server_instance_at_head.object_store_creator) | 184 server_instance_at_head.object_store_creator) |
| 198 | 185 |
| 199 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): | 186 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): |
| 200 # TODO(kalman): return a new ServerInstance at an explicit revision in | 187 # TODO(kalman): return a new ServerInstance at an explicit revision in |
| 201 # case the HEAD version changes underneath us. | 188 # case the HEAD version changes underneath us. |
| 202 return server_instance_at_head | 189 return server_instance_at_head |
| 203 | 190 |
| 204 # The version in app.yaml is greater than the currently running app's. | 191 # The version in app.yaml is greater than the currently running app's. |
| 205 # The safe version is the one before it changed. | 192 # The safe version is the one before it changed. |
| 206 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( | 193 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( |
| 207 delegate.GetAppVersion()) - 1 | 194 delegate.GetAppVersion()) - 1 |
| 208 | 195 |
| 209 logging.info('cron/%s: app version %s is out of date, safe is %s' % ( | 196 logging.info('cron: app version %s is out of date, safe is %s' % ( |
| 210 channel, delegate.GetAppVersion(), safe_revision)) | 197 delegate.GetAppVersion(), safe_revision)) |
| 211 | 198 |
| 212 return self._CreateServerInstance(channel, safe_revision) | 199 return self._CreateServerInstance(safe_revision) |
| 213 | 200 |
| 214 def _CreateObjectStoreCreator(self, channel): | 201 def _CreateObjectStoreCreator(self): |
| 215 return ObjectStoreCreator(channel, start_empty=True) | 202 return ObjectStoreCreator(start_empty=True) |
| 216 | 203 |
| 217 def _GetBranchForChannel(self, channel): | 204 def _CreateServerInstance(self, revision): |
| 218 object_store_creator = self._CreateObjectStoreCreator(channel) | 205 object_store_creator = self._CreateObjectStoreCreator() |
| 219 return (self._delegate.CreateBranchUtility(object_store_creator) | |
| 220 .GetBranchForChannel(channel)) | |
| 221 | |
| 222 def _CreateServerInstance(self, channel, revision): | |
| 223 object_store_creator = self._CreateObjectStoreCreator(channel) | |
| 224 host_file_system = CachingFileSystem( | 206 host_file_system = CachingFileSystem( |
| 225 self._delegate.CreateHostFileSystemForBranchAndRevision( | 207 self._delegate.CreateHostFileSystemForRevision(revision), |
| 226 self._GetBranchForChannel(channel), | |
| 227 revision), | |
| 228 object_store_creator) | 208 object_store_creator) |
| 229 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem( | 209 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem( |
| 230 object_store_creator) | 210 object_store_creator) |
| 231 compiled_host_fs_factory = CompiledFileSystem.Factory( | 211 compiled_host_fs_factory = CompiledFileSystem.Factory( |
| 232 host_file_system, | 212 host_file_system, |
| 233 object_store_creator) | 213 object_store_creator) |
| 234 return ServerInstance(channel, | 214 return ServerInstance(object_store_creator, |
| 235 object_store_creator, | |
| 236 host_file_system, | 215 host_file_system, |
| 237 app_samples_file_system, | 216 app_samples_file_system, |
| 238 '/static' if channel == 'stable' else | 217 compiled_host_fs_factory, |
| 239 '/%s/static' % channel, | 218 '/static') |
| 240 compiled_host_fs_factory) | |
| OLD | NEW |