Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(620)

Side by Side Diff: chrome/common/extensions/docs/server2/cron_servlet.py

Issue 15087006: Docserver: there is only one. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: epic rebase Created 7 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright 2013 The Chromium Authors. All rights reserved. 1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import logging 5 import logging
6 import time 6 import time
7 import traceback 7 import traceback
8 8
9 from app_yaml_helper import AppYamlHelper 9 from app_yaml_helper import AppYamlHelper
10 from appengine_wrappers import ( 10 from appengine_wrappers import (
11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice) 11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice)
12 from branch_utility import BranchUtility 12 from branch_utility import BranchUtility
13 from compiled_file_system import CompiledFileSystem 13 from compiled_file_system import CompiledFileSystem
14 from empty_dir_file_system import EmptyDirFileSystem 14 from empty_dir_file_system import EmptyDirFileSystem
15 from file_system_util import CreateURLsFromPaths 15 from file_system_util import CreateURLsFromPaths
16 from github_file_system import GithubFileSystem 16 from github_file_system import GithubFileSystem
17 from host_file_system_creator import HostFileSystemCreator 17 from host_file_system_creator import HostFileSystemCreator
18 from object_store_creator import ObjectStoreCreator 18 from object_store_creator import ObjectStoreCreator
19 from render_servlet import RenderServlet 19 from render_servlet import RenderServlet
20 from server_instance import ServerInstance 20 from server_instance import ServerInstance
21 from servlet import Servlet, Request, Response 21 from servlet import Servlet, Request, Response
22 import svn_constants 22 import svn_constants
23 23
24 class _SingletonRenderServletDelegate(RenderServlet.Delegate): 24 class _SingletonRenderServletDelegate(RenderServlet.Delegate):
25 def __init__(self, server_instance): 25 def __init__(self, server_instance):
26 self._server_instance = server_instance 26 self._server_instance = server_instance
27 27
28 def CreateServerInstanceForChannel(self, channel): 28 def CreateServerInstance(self):
29 return self._server_instance 29 return self._server_instance
30 30
31 class CronServlet(Servlet): 31 class CronServlet(Servlet):
32 '''Servlet which runs a cron job. 32 '''Servlet which runs a cron job.
33 ''' 33 '''
34 def __init__(self, request, delegate_for_test=None): 34 def __init__(self, request, delegate_for_test=None):
35 Servlet.__init__(self, request) 35 Servlet.__init__(self, request)
36 self._channel = request.path.strip('/')
37 self._delegate = delegate_for_test or CronServlet.Delegate() 36 self._delegate = delegate_for_test or CronServlet.Delegate()
38 37
39 class Delegate(object): 38 class Delegate(object):
40 '''CronServlet's runtime dependencies. Override for testing. 39 '''CronServlet's runtime dependencies. Override for testing.
41 ''' 40 '''
42 def CreateBranchUtility(self, object_store_creator): 41 def CreateBranchUtility(self, object_store_creator):
43 return BranchUtility.Create(object_store_creator) 42 return BranchUtility.Create(object_store_creator)
44 43
45 def CreateHostFileSystemCreator(self, object_store_creator): 44 def CreateHostFileSystemCreator(self, object_store_creator):
46 return HostFileSystemCreator(object_store_creator) 45 return HostFileSystemCreator(object_store_creator)
(...skipping 16 matching lines...) Expand all
63 finally: 62 finally:
64 logservice.flush() 63 logservice.flush()
65 64
66 def _GetImpl(self): 65 def _GetImpl(self):
67 # Cron strategy: 66 # Cron strategy:
68 # 67 #
69 # Find all public template files and static files, and render them. Most of 68 # Find all public template files and static files, and render them. Most of
70 # the time these won't have changed since the last cron run, so it's a 69 # the time these won't have changed since the last cron run, so it's a
71 # little wasteful, but hopefully rendering is really fast (if it isn't we 70 # little wasteful, but hopefully rendering is really fast (if it isn't we
72 # have a problem). 71 # have a problem).
73 channel = self._channel 72 logging.info('cron: starting')
74 logging.info('cron/%s: starting' % channel)
75 73
76 # This is returned every time RenderServlet wants to create a new 74 # This is returned every time RenderServlet wants to create a new
77 # ServerInstance. 75 # ServerInstance.
78 server_instance = self._GetSafeServerInstance() 76 server_instance = self._GetSafeServerInstance()
79 77
80 def get_via_render_servlet(path): 78 def get_via_render_servlet(path):
81 request = Request(path, self._request.host, self._request.headers) 79 request = Request(path, self._request.host, self._request.headers)
82 delegate = _SingletonRenderServletDelegate(server_instance) 80 delegate = _SingletonRenderServletDelegate(server_instance)
83 return RenderServlet(request, delegate).Get() 81 return RenderServlet(request, delegate).Get()
84 82
85 def run_cron_for_dir(d, path_prefix=''): 83 def run_cron_for_dir(d, path_prefix=''):
86 success = True 84 success = True
87 start_time = time.time() 85 start_time = time.time()
88 files = dict( 86 files = dict(
89 CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix)) 87 CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix))
90 logging.info('cron/%s: rendering %s files from %s...' % ( 88 logging.info('cron: rendering %s files from %s...' % (len(files), d))
91 channel, len(files), d))
92 try: 89 try:
93 for i, path in enumerate(files): 90 for i, path in enumerate(files):
94 error = None 91 error = None
95 try: 92 try:
96 response = get_via_render_servlet(path) 93 response = get_via_render_servlet(path)
97 if response.status != 200: 94 if response.status != 200:
98 error = 'Got %s response' % response.status 95 error = 'Got %s response' % response.status
99 except DeadlineExceededError: 96 except DeadlineExceededError:
100 logging.error( 97 logging.error(
101 'cron/%s: deadline exceeded rendering %s (%s of %s): %s' % ( 98 'cron: deadline exceeded rendering %s (%s of %s): %s' % (
102 channel, path, i + 1, len(files), traceback.format_exc())) 99 path, i + 1, len(files), traceback.format_exc()))
103 raise 100 raise
104 except error: 101 except error:
105 pass 102 pass
106 if error: 103 if error:
107 logging.error('cron/%s: error rendering %s: %s' % ( 104 logging.error('cron: error rendering %s: %s' % (path, error))
108 channel, path, error))
109 success = False 105 success = False
110 finally: 106 finally:
111 logging.info('cron/%s: rendering %s files from %s took %s seconds' % ( 107 logging.info('cron: rendering %s files from %s took %s seconds' % (
112 channel, len(files), d, time.time() - start_time)) 108 len(files), d, time.time() - start_time))
113 return success 109 return success
114 110
115 success = True 111 success = True
116 try: 112 try:
117 # Render all of the publicly accessible files. 113 # Render all of the publicly accessible files.
118 cron_runs = [ 114 cron_runs = [
119 # Note: rendering the public templates will pull in all of the private 115 # Note: rendering the public templates will pull in all of the private
120 # templates. 116 # templates.
121 (svn_constants.PUBLIC_TEMPLATE_PATH, ''), 117 (svn_constants.PUBLIC_TEMPLATE_PATH, ''),
122 # Note: rendering the public templates will have pulled in the .js 118 # Note: rendering the public templates will have pulled in the .js
(...skipping 17 matching lines...) Expand all
140 136
141 # Extension examples have zip files too. Well, so do apps, but the app 137 # Extension examples have zip files too. Well, so do apps, but the app
142 # file system doesn't get the Offline treatment so they don't need cron. 138 # file system doesn't get the Offline treatment so they don't need cron.
143 if not IsDevServer(): 139 if not IsDevServer():
144 manifest_json = '/manifest.json' 140 manifest_json = '/manifest.json'
145 example_zips = [ 141 example_zips = [
146 '%s.zip' % filename[:-len(manifest_json)] 142 '%s.zip' % filename[:-len(manifest_json)]
147 for filename in server_instance.content_cache.GetFromFileListing( 143 for filename in server_instance.content_cache.GetFromFileListing(
148 svn_constants.EXAMPLES_PATH) 144 svn_constants.EXAMPLES_PATH)
149 if filename.endswith(manifest_json)] 145 if filename.endswith(manifest_json)]
150 logging.info('cron/%s: rendering %s example zips...' % ( 146 logging.info('cron: rendering %s example zips...' % len(example_zips))
151 channel, len(example_zips)))
152 start_time = time.time() 147 start_time = time.time()
153 try: 148 try:
154 success = success and all( 149 success = success and all(
155 get_via_render_servlet('extensions/examples/%s' % z).status == 200 150 get_via_render_servlet('extensions/examples/%s' % z).status == 200
156 for z in example_zips) 151 for z in example_zips)
157 finally: 152 finally:
158 logging.info('cron/%s: rendering %s example zips took %s seconds' % ( 153 logging.info('cron: rendering %s example zips took %s seconds' % (
159 channel, len(example_zips), time.time() - start_time)) 154 len(example_zips), time.time() - start_time))
160 155
161 except DeadlineExceededError: 156 except DeadlineExceededError:
162 success = False 157 success = False
163 158
164 logging.info("cron/%s: running Redirector cron..." % channel) 159 logging.info('cron: running Redirector cron...')
165 server_instance.redirector.Cron() 160 server_instance.redirector.Cron()
166 161
167 logging.info('cron/%s: finished' % channel) 162 logging.info('cron: finished (%s)' % ('success' if success else 'failure',))
168 163
169 return (Response.Ok('Success') if success else 164 return (Response.Ok('Success') if success else
170 Response.InternalError('Failure')) 165 Response.InternalError('Failure'))
171 166
172 def _GetSafeServerInstance(self): 167 def _GetSafeServerInstance(self):
173 '''Returns a ServerInstance with a host file system at a safe revision, 168 '''Returns a ServerInstance with a host file system at a safe revision,
174 meaning the last revision that the current running version of the server 169 meaning the last revision that the current running version of the server
175 existed. 170 existed.
176 ''' 171 '''
177 channel = self._channel
178 delegate = self._delegate 172 delegate = self._delegate
179 server_instance_at_head = self._CreateServerInstance(channel, None) 173 server_instance_at_head = self._CreateServerInstance(None)
180 174
181 app_yaml_handler = AppYamlHelper( 175 app_yaml_handler = AppYamlHelper(
182 svn_constants.APP_YAML_PATH, 176 svn_constants.APP_YAML_PATH,
183 server_instance_at_head.host_file_system, 177 server_instance_at_head.host_file_system,
184 server_instance_at_head.object_store_creator, 178 server_instance_at_head.object_store_creator,
185 server_instance_at_head.host_file_system_creator, 179 server_instance_at_head.host_file_system_creator)
186 self._GetBranchForChannel(channel))
187 180
188 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): 181 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
189 # TODO(kalman): return a new ServerInstance at an explicit revision in 182 # TODO(kalman): return a new ServerInstance at an explicit revision in
190 # case the HEAD version changes underneath us. 183 # case the HEAD version changes underneath us.
191 return server_instance_at_head 184 return server_instance_at_head
192 185
193 # The version in app.yaml is greater than the currently running app's. 186 # The version in app.yaml is greater than the currently running app's.
194 # The safe version is the one before it changed. 187 # The safe version is the one before it changed.
195 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( 188 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
196 delegate.GetAppVersion()) - 1 189 delegate.GetAppVersion()) - 1
197 190
198 logging.info('cron/%s: app version %s is out of date, safe is %s' % ( 191 logging.info('cron: app version %s is out of date, safe is %s' % (
199 channel, delegate.GetAppVersion(), safe_revision)) 192 delegate.GetAppVersion(), safe_revision))
200 193
201 return self._CreateServerInstance(channel, safe_revision) 194 return self._CreateServerInstance(safe_revision)
202 195
203 def _CreateObjectStoreCreator(self, channel): 196 def _CreateServerInstance(self, revision):
204 return ObjectStoreCreator(channel, start_empty=True) 197 object_store_creator = ObjectStoreCreator(start_empty=True)
205
206 def _GetBranchForChannel(self, channel):
207 object_store_creator = self._CreateObjectStoreCreator(channel)
208 return (self._delegate.CreateBranchUtility(object_store_creator)
209 .GetChannelInfo(channel).branch)
210
211 def _CreateServerInstance(self, channel, revision):
212 object_store_creator = self._CreateObjectStoreCreator(channel)
213 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) 198 branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
214 host_file_system_creator = self._delegate.CreateHostFileSystemCreator( 199 host_file_system_creator = self._delegate.CreateHostFileSystemCreator(
215 object_store_creator) 200 object_store_creator)
216 host_file_system = host_file_system_creator.Create( 201 host_file_system = host_file_system_creator.Create(revision=revision)
217 branch_utility.GetChannelInfo(channel).branch,
218 revision=revision)
219 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem( 202 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem(
220 object_store_creator) 203 object_store_creator)
221 compiled_host_fs_factory = CompiledFileSystem.Factory( 204 compiled_host_fs_factory = CompiledFileSystem.Factory(
222 host_file_system, 205 host_file_system,
223 object_store_creator) 206 object_store_creator)
224 return ServerInstance(channel, 207 return ServerInstance(object_store_creator,
225 object_store_creator,
226 host_file_system, 208 host_file_system,
227 app_samples_file_system, 209 app_samples_file_system,
228 '' if channel == 'stable' else '/%s' % channel, 210 '',
229 compiled_host_fs_factory, 211 compiled_host_fs_factory,
230 branch_utility, 212 branch_utility,
231 host_file_system_creator) 213 host_file_system_creator)
OLDNEW
« no previous file with comments | « chrome/common/extensions/docs/server2/cron.yaml ('k') | chrome/common/extensions/docs/server2/cron_servlet_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698