Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(634)

Side by Side Diff: chrome/common/extensions/docs/server2/cron_servlet.py

Issue 15087006: Docserver: there is only one. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: better redirects Created 7 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright 2013 The Chromium Authors. All rights reserved. 1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import logging 5 import logging
6 import time 6 import time
7 import traceback 7 import traceback
8 8
9 from app_yaml_helper import AppYamlHelper 9 from app_yaml_helper import AppYamlHelper
10 from appengine_wrappers import ( 10 from appengine_wrappers import (
11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice) 11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice)
12 from branch_utility import BranchUtility
13 from caching_file_system import CachingFileSystem 12 from caching_file_system import CachingFileSystem
14 from compiled_file_system import CompiledFileSystem 13 from compiled_file_system import CompiledFileSystem
15 from empty_dir_file_system import EmptyDirFileSystem 14 from empty_dir_file_system import EmptyDirFileSystem
16 from github_file_system import GithubFileSystem 15 from github_file_system import GithubFileSystem
17 from object_store_creator import ObjectStoreCreator 16 from object_store_creator import ObjectStoreCreator
18 from render_servlet import RenderServlet 17 from render_servlet import RenderServlet
19 from server_instance import ServerInstance 18 from server_instance import ServerInstance
20 from servlet import Servlet, Request, Response 19 from servlet import Servlet, Request, Response
21 from subversion_file_system import SubversionFileSystem 20 from subversion_file_system import SubversionFileSystem
22 import svn_constants 21 import svn_constants
23 from third_party.json_schema_compiler.memoize import memoize 22 from third_party.json_schema_compiler.memoize import memoize
24 23
25 class _SingletonRenderServletDelegate(RenderServlet.Delegate): 24 class _SingletonRenderServletDelegate(RenderServlet.Delegate):
26 def __init__(self, server_instance): 25 def __init__(self, server_instance):
27 self._server_instance = server_instance 26 self._server_instance = server_instance
28 27
29 def CreateServerInstanceForChannel(self, channel): 28 def CreateServerInstance(self):
30 return self._server_instance 29 return self._server_instance
31 30
32 class CronServlet(Servlet): 31 class CronServlet(Servlet):
33 '''Servlet which runs a cron job. 32 '''Servlet which runs a cron job.
34 ''' 33 '''
35 def __init__(self, request, delegate_for_test=None): 34 def __init__(self, request, delegate_for_test=None):
36 Servlet.__init__(self, request) 35 Servlet.__init__(self, request)
37 self._channel = request.path.strip('/')
38 self._delegate = delegate_for_test or CronServlet.Delegate() 36 self._delegate = delegate_for_test or CronServlet.Delegate()
39 37
40 class Delegate(object): 38 class Delegate(object):
41 '''CronServlet's runtime dependencies. Override for testing. 39 '''CronServlet's runtime dependencies. Override for testing.
42 ''' 40 '''
43 def CreateBranchUtility(self, object_store_creator): 41 def CreateHostFileSystemForRevision(self, revision):
44 return BranchUtility.Create(object_store_creator) 42 return SubversionFileSystem.Create('trunk', revision=revision)
45
46 def CreateHostFileSystemForBranchAndRevision(self, branch, revision):
47 return SubversionFileSystem.Create(branch, revision=revision)
48 43
49 def CreateAppSamplesFileSystem(self, object_store_creator): 44 def CreateAppSamplesFileSystem(self, object_store_creator):
50 # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's 45 # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's
51 # not supported yet (see comment there). 46 # not supported yet (see comment there).
52 return (EmptyDirFileSystem() if IsDevServer() else 47 return (EmptyDirFileSystem() if IsDevServer() else
53 GithubFileSystem.Create(object_store_creator)) 48 GithubFileSystem.Create(object_store_creator))
54 49
55 def GetAppVersion(self): 50 def GetAppVersion(self):
56 return GetAppVersion() 51 return GetAppVersion()
57 52
58 def Get(self): 53 def Get(self):
59 # Crons often time out, and when they do *and* then eventually try to 54 # Crons often time out, and when they do *and* then eventually try to
60 # flush logs they die. Turn off autoflush and manually do so at the end. 55 # flush logs they die. Turn off autoflush and manually do so at the end.
61 logservice.AUTOFLUSH_ENABLED = False 56 logservice.AUTOFLUSH_ENABLED = False
62 try: 57 try:
63 return self._GetImpl() 58 return self._GetImpl()
64 finally: 59 finally:
65 logservice.flush() 60 logservice.flush()
66 61
67 def _GetImpl(self): 62 def _GetImpl(self):
68 # Cron strategy: 63 # Cron strategy:
69 # 64 #
70 # Find all public template files and static files, and render them. Most of 65 # Find all public template files and static files, and render them. Most of
71 # the time these won't have changed since the last cron run, so it's a 66 # the time these won't have changed since the last cron run, so it's a
72 # little wasteful, but hopefully rendering is really fast (if it isn't we 67 # little wasteful, but hopefully rendering is really fast (if it isn't we
73 # have a problem). 68 # have a problem).
74 channel = self._channel 69 logging.info('cron: starting')
75 logging.info('cron/%s: starting' % channel)
76 70
77 # This is returned every time RenderServlet wants to create a new 71 # This is returned every time RenderServlet wants to create a new
78 # ServerInstance. 72 # ServerInstance.
79 server_instance = self._GetSafeServerInstance() 73 server_instance = self._GetSafeServerInstance()
80 74
81 def get_via_render_servlet(path): 75 def get_via_render_servlet(path):
82 return RenderServlet( 76 return RenderServlet(
83 Request(path, self._request.host, self._request.headers), 77 Request(path, self._request.host, self._request.headers),
84 _SingletonRenderServletDelegate(server_instance)).Get() 78 _SingletonRenderServletDelegate(server_instance)).Get()
85 79
86 def run_cron_for_dir(d, path_prefix=''): 80 def run_cron_for_dir(d, path_prefix=''):
87 success = True 81 success = True
88 start_time = time.time() 82 start_time = time.time()
89 files = [f for f in server_instance.content_cache.GetFromFileListing(d) 83 files = [f for f in server_instance.content_cache.GetFromFileListing(d)
90 if not f.endswith('/')] 84 if not f.endswith('/')]
91 logging.info('cron/%s: rendering %s files from %s...' % ( 85 logging.info('cron: rendering %s files from %s...' % (len(files), d))
92 channel, len(files), d))
93 try: 86 try:
94 for i, f in enumerate(files): 87 for i, f in enumerate(files):
95 error = None 88 error = None
96 path = '%s%s' % (path_prefix, f) 89 path = '%s%s' % (path_prefix, f)
97 try: 90 try:
98 response = get_via_render_servlet(path) 91 response = get_via_render_servlet(path)
99 if response.status != 200: 92 if response.status != 200:
100 error = 'Got %s response' % response.status 93 error = 'Got %s response' % response.status
101 except DeadlineExceededError: 94 except DeadlineExceededError:
102 logging.error( 95 logging.error(
103 'cron/%s: deadline exceeded rendering %s (%s of %s): %s' % ( 96 'cron: deadline exceeded rendering %s (%s of %s): %s' % (
104 channel, path, i + 1, len(files), traceback.format_exc())) 97 path, i + 1, len(files), traceback.format_exc()))
105 raise 98 raise
106 except error: 99 except error:
107 pass 100 pass
108 if error: 101 if error:
109 logging.error('cron/%s: error rendering %s: %s' % ( 102 logging.error('cron: error rendering %s: %s' % (path, error))
110 channel, path, error))
111 success = False 103 success = False
112 finally: 104 finally:
113 logging.info('cron/%s: rendering %s files from %s took %s seconds' % ( 105 logging.info('cron: rendering %s files from %s took %s seconds' % (
114 channel, len(files), d, time.time() - start_time)) 106 len(files), d, time.time() - start_time))
115 return success 107 return success
116 108
117 success = True 109 success = True
118 try: 110 try:
119 # Render all of the publicly accessible files. 111 # Render all of the publicly accessible files.
120 cron_runs = [ 112 cron_runs = [
121 # Note: rendering the public templates will pull in all of the private 113 # Note: rendering the public templates will pull in all of the private
122 # templates. 114 # templates.
123 (svn_constants.PUBLIC_TEMPLATE_PATH, ''), 115 (svn_constants.PUBLIC_TEMPLATE_PATH, ''),
124 # Note: rendering the public templates will have pulled in the .js 116 # Note: rendering the public templates will have pulled in the .js
(...skipping 17 matching lines...) Expand all
142 134
143 # Extension examples have zip files too. Well, so do apps, but the app 135 # Extension examples have zip files too. Well, so do apps, but the app
144 # file system doesn't get the Offline treatment so they don't need cron. 136 # file system doesn't get the Offline treatment so they don't need cron.
145 if not IsDevServer(): 137 if not IsDevServer():
146 manifest_json = '/manifest.json' 138 manifest_json = '/manifest.json'
147 example_zips = [ 139 example_zips = [
148 '%s.zip' % filename[:-len(manifest_json)] 140 '%s.zip' % filename[:-len(manifest_json)]
149 for filename in server_instance.content_cache.GetFromFileListing( 141 for filename in server_instance.content_cache.GetFromFileListing(
150 svn_constants.EXAMPLES_PATH) 142 svn_constants.EXAMPLES_PATH)
151 if filename.endswith(manifest_json)] 143 if filename.endswith(manifest_json)]
152 logging.info('cron/%s: rendering %s example zips...' % ( 144 logging.info('cron: rendering %s example zips...' % len(example_zips))
153 channel, len(example_zips)))
154 start_time = time.time() 145 start_time = time.time()
155 try: 146 try:
156 success = success and all( 147 success = success and all(
157 get_via_render_servlet('extensions/examples/%s' % z).status == 200 148 get_via_render_servlet('extensions/examples/%s' % z).status == 200
158 for z in example_zips) 149 for z in example_zips)
159 finally: 150 finally:
160 logging.info('cron/%s: rendering %s example zips took %s seconds' % ( 151 logging.info('cron: rendering %s example zips took %s seconds' % (
161 channel, len(example_zips), time.time() - start_time)) 152 len(example_zips), time.time() - start_time))
162 153
163 # Also trigger a redirect so that PathCanonicalizer has an opportunity to 154 # Also trigger a redirect so that PathCanonicalizer has an opportunity to
164 # cache file listings. 155 # cache file listings.
165 logging.info('cron/%s: triggering a redirect...' % channel) 156 logging.info('cron: triggering a redirect...')
166 redirect_response = get_via_render_servlet('storage.html') 157 redirect_response = get_via_render_servlet('storage.html')
167 success = success and redirect_response.status == 302 158 success = success and redirect_response.status == 302
168 except DeadlineExceededError: 159 except DeadlineExceededError:
169 success = False 160 success = False
170 161
171 logging.info('cron/%s: finished' % channel) 162 logging.info('cron: finished (%s)' % 'success' if success else 'failure')
172 163
173 return (Response.Ok('Success') if success else 164 return (Response.Ok('Success') if success else
174 Response.InternalError('Failure')) 165 Response.InternalError('Failure'))
175 166
176 def _GetSafeServerInstance(self): 167 def _GetSafeServerInstance(self):
177 '''Returns a ServerInstance with a host file system at a safe revision, 168 '''Returns a ServerInstance with a host file system at a safe revision,
178 meaning the last revision that the current running version of the server 169 meaning the last revision that the current running version of the server
179 existed. 170 existed.
180 ''' 171 '''
181 channel = self._channel
182 delegate = self._delegate 172 delegate = self._delegate
183 173
184 server_instance_at_head = self._CreateServerInstance(channel, None) 174 server_instance_at_head = self._CreateServerInstance(None)
185 175
186 get_branch_for_channel = self._GetBranchForChannel
187 class AppYamlHelperDelegate(AppYamlHelper.Delegate): 176 class AppYamlHelperDelegate(AppYamlHelper.Delegate):
188 def GetHostFileSystemForRevision(self, revision): 177 def GetHostFileSystemForRevision(self, revision):
189 return delegate.CreateHostFileSystemForBranchAndRevision( 178 return delegate.CreateHostFileSystemForRevision(revision)
190 get_branch_for_channel(channel),
191 revision)
192 179
193 app_yaml_handler = AppYamlHelper( 180 app_yaml_handler = AppYamlHelper(
194 svn_constants.APP_YAML_PATH, 181 svn_constants.APP_YAML_PATH,
195 server_instance_at_head.host_file_system, 182 server_instance_at_head.host_file_system,
196 AppYamlHelperDelegate(), 183 AppYamlHelperDelegate(),
197 server_instance_at_head.object_store_creator) 184 server_instance_at_head.object_store_creator)
198 185
199 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): 186 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
200 # TODO(kalman): return a new ServerInstance at an explicit revision in 187 # TODO(kalman): return a new ServerInstance at an explicit revision in
201 # case the HEAD version changes underneath us. 188 # case the HEAD version changes underneath us.
202 return server_instance_at_head 189 return server_instance_at_head
203 190
204 # The version in app.yaml is greater than the currently running app's. 191 # The version in app.yaml is greater than the currently running app's.
205 # The safe version is the one before it changed. 192 # The safe version is the one before it changed.
206 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( 193 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
207 delegate.GetAppVersion()) - 1 194 delegate.GetAppVersion()) - 1
208 195
209 logging.info('cron/%s: app version %s is out of date, safe is %s' % ( 196 logging.info('cron: app version %s is out of date, safe is %s' % (
210 channel, delegate.GetAppVersion(), safe_revision)) 197 delegate.GetAppVersion(), safe_revision))
211 198
212 return self._CreateServerInstance(channel, safe_revision) 199 return self._CreateServerInstance(safe_revision)
213 200
214 def _CreateObjectStoreCreator(self, channel): 201 def _CreateObjectStoreCreator(self):
215 return ObjectStoreCreator(channel, start_empty=True) 202 return ObjectStoreCreator(start_empty=True)
216 203
217 def _GetBranchForChannel(self, channel): 204 def _CreateServerInstance(self, revision):
218 object_store_creator = self._CreateObjectStoreCreator(channel) 205 object_store_creator = self._CreateObjectStoreCreator()
219 return (self._delegate.CreateBranchUtility(object_store_creator)
220 .GetBranchForChannel(channel))
221
222 def _CreateServerInstance(self, channel, revision):
223 object_store_creator = self._CreateObjectStoreCreator(channel)
224 host_file_system = CachingFileSystem( 206 host_file_system = CachingFileSystem(
225 self._delegate.CreateHostFileSystemForBranchAndRevision( 207 self._delegate.CreateHostFileSystemForRevision(revision),
226 self._GetBranchForChannel(channel),
227 revision),
228 object_store_creator) 208 object_store_creator)
229 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem( 209 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem(
230 object_store_creator) 210 object_store_creator)
231 compiled_host_fs_factory = CompiledFileSystem.Factory( 211 compiled_host_fs_factory = CompiledFileSystem.Factory(
232 host_file_system, 212 host_file_system,
233 object_store_creator) 213 object_store_creator)
234 return ServerInstance(channel, 214 return ServerInstance(object_store_creator,
235 object_store_creator,
236 host_file_system, 215 host_file_system,
237 app_samples_file_system, 216 app_samples_file_system,
238 '/static' if channel == 'stable' else 217 compiled_host_fs_factory,
239 '/%s/static' % channel, 218 '/static')
240 compiled_host_fs_factory)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698