OLD | NEW |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import logging | 5 import logging |
6 import time | 6 import time |
7 import traceback | 7 import traceback |
8 | 8 |
9 from app_yaml_helper import AppYamlHelper | 9 from app_yaml_helper import AppYamlHelper |
10 from appengine_wrappers import ( | 10 from appengine_wrappers import ( |
11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice) | 11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice) |
12 from branch_utility import BranchUtility | 12 from branch_utility import BranchUtility |
13 from compiled_file_system import CompiledFileSystem | 13 from compiled_file_system import CompiledFileSystem |
14 from empty_dir_file_system import EmptyDirFileSystem | 14 from empty_dir_file_system import EmptyDirFileSystem |
15 from file_system_util import CreateURLsFromPaths | 15 from file_system_util import CreateURLsFromPaths |
16 from github_file_system import GithubFileSystem | 16 from github_file_system import GithubFileSystem |
17 from host_file_system_creator import HostFileSystemCreator | 17 from host_file_system_creator import HostFileSystemCreator |
18 from object_store_creator import ObjectStoreCreator | 18 from object_store_creator import ObjectStoreCreator |
19 from render_servlet import RenderServlet | 19 from render_servlet import RenderServlet |
20 from server_instance import ServerInstance | 20 from server_instance import ServerInstance |
21 from servlet import Servlet, Request, Response | 21 from servlet import Servlet, Request, Response |
22 import svn_constants | 22 import svn_constants |
23 | 23 |
24 class _SingletonRenderServletDelegate(RenderServlet.Delegate): | 24 class _SingletonRenderServletDelegate(RenderServlet.Delegate): |
25 def __init__(self, server_instance): | 25 def __init__(self, server_instance): |
26 self._server_instance = server_instance | 26 self._server_instance = server_instance |
27 | 27 |
28 def CreateServerInstanceForChannel(self, channel): | 28 def CreateServerInstance(self): |
29 return self._server_instance | 29 return self._server_instance |
30 | 30 |
31 class CronServlet(Servlet): | 31 class CronServlet(Servlet): |
32 '''Servlet which runs a cron job. | 32 '''Servlet which runs a cron job. |
33 ''' | 33 ''' |
34 def __init__(self, request, delegate_for_test=None): | 34 def __init__(self, request, delegate_for_test=None): |
35 Servlet.__init__(self, request) | 35 Servlet.__init__(self, request) |
36 self._channel = request.path.strip('/') | |
37 self._delegate = delegate_for_test or CronServlet.Delegate() | 36 self._delegate = delegate_for_test or CronServlet.Delegate() |
38 | 37 |
39 class Delegate(object): | 38 class Delegate(object): |
40 '''CronServlet's runtime dependencies. Override for testing. | 39 '''CronServlet's runtime dependencies. Override for testing. |
41 ''' | 40 ''' |
42 def CreateBranchUtility(self, object_store_creator): | 41 def CreateBranchUtility(self, object_store_creator): |
43 return BranchUtility.Create(object_store_creator) | 42 return BranchUtility.Create(object_store_creator) |
44 | 43 |
45 def CreateHostFileSystemCreator(self, object_store_creator): | 44 def CreateHostFileSystemCreator(self, object_store_creator): |
46 return HostFileSystemCreator(object_store_creator) | 45 return HostFileSystemCreator(object_store_creator) |
(...skipping 16 matching lines...) Expand all Loading... |
63 finally: | 62 finally: |
64 logservice.flush() | 63 logservice.flush() |
65 | 64 |
66 def _GetImpl(self): | 65 def _GetImpl(self): |
67 # Cron strategy: | 66 # Cron strategy: |
68 # | 67 # |
69 # Find all public template files and static files, and render them. Most of | 68 # Find all public template files and static files, and render them. Most of |
70 # the time these won't have changed since the last cron run, so it's a | 69 # the time these won't have changed since the last cron run, so it's a |
71 # little wasteful, but hopefully rendering is really fast (if it isn't we | 70 # little wasteful, but hopefully rendering is really fast (if it isn't we |
72 # have a problem). | 71 # have a problem). |
73 channel = self._channel | 72 logging.info('cron: starting') |
74 logging.info('cron/%s: starting' % channel) | |
75 | 73 |
76 # This is returned every time RenderServlet wants to create a new | 74 # This is returned every time RenderServlet wants to create a new |
77 # ServerInstance. | 75 # ServerInstance. |
78 server_instance = self._GetSafeServerInstance() | 76 server_instance = self._GetSafeServerInstance() |
79 | 77 |
80 def get_via_render_servlet(path): | 78 def get_via_render_servlet(path): |
81 request = Request(path, self._request.host, self._request.headers) | 79 request = Request(path, self._request.host, self._request.headers) |
82 delegate = _SingletonRenderServletDelegate(server_instance) | 80 delegate = _SingletonRenderServletDelegate(server_instance) |
83 return RenderServlet(request, delegate).Get() | 81 return RenderServlet(request, delegate).Get() |
84 | 82 |
85 def run_cron_for_dir(d, path_prefix=''): | 83 def run_cron_for_dir(d, path_prefix=''): |
86 success = True | 84 success = True |
87 start_time = time.time() | 85 start_time = time.time() |
88 files = dict( | 86 files = dict( |
89 CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix)) | 87 CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix)) |
90 logging.info('cron/%s: rendering %s files from %s...' % ( | 88 logging.info('cron: rendering %s files from %s...' % (len(files), d)) |
91 channel, len(files), d)) | |
92 try: | 89 try: |
93 for i, path in enumerate(files): | 90 for i, path in enumerate(files): |
94 error = None | 91 error = None |
95 try: | 92 try: |
96 response = get_via_render_servlet(path) | 93 response = get_via_render_servlet(path) |
97 if response.status != 200: | 94 if response.status != 200: |
98 error = 'Got %s response' % response.status | 95 error = 'Got %s response' % response.status |
99 except DeadlineExceededError: | 96 except DeadlineExceededError: |
100 logging.error( | 97 logging.error( |
101 'cron/%s: deadline exceeded rendering %s (%s of %s): %s' % ( | 98 'cron: deadline exceeded rendering %s (%s of %s): %s' % ( |
102 channel, path, i + 1, len(files), traceback.format_exc())) | 99 path, i + 1, len(files), traceback.format_exc())) |
103 raise | 100 raise |
104 except error: | 101 except error: |
105 pass | 102 pass |
106 if error: | 103 if error: |
107 logging.error('cron/%s: error rendering %s: %s' % ( | 104 logging.error('cron: error rendering %s: %s' % (path, error)) |
108 channel, path, error)) | |
109 success = False | 105 success = False |
110 finally: | 106 finally: |
111 logging.info('cron/%s: rendering %s files from %s took %s seconds' % ( | 107 logging.info('cron: rendering %s files from %s took %s seconds' % ( |
112 channel, len(files), d, time.time() - start_time)) | 108 len(files), d, time.time() - start_time)) |
113 return success | 109 return success |
114 | 110 |
115 success = True | 111 success = True |
116 try: | 112 try: |
117 # Render all of the publicly accessible files. | 113 # Render all of the publicly accessible files. |
118 cron_runs = [ | 114 cron_runs = [ |
119 # Note: rendering the public templates will pull in all of the private | 115 # Note: rendering the public templates will pull in all of the private |
120 # templates. | 116 # templates. |
121 (svn_constants.PUBLIC_TEMPLATE_PATH, ''), | 117 (svn_constants.PUBLIC_TEMPLATE_PATH, ''), |
122 # Note: rendering the public templates will have pulled in the .js | 118 # Note: rendering the public templates will have pulled in the .js |
(...skipping 17 matching lines...) Expand all Loading... |
140 | 136 |
141 # Extension examples have zip files too. Well, so do apps, but the app | 137 # Extension examples have zip files too. Well, so do apps, but the app |
142 # file system doesn't get the Offline treatment so they don't need cron. | 138 # file system doesn't get the Offline treatment so they don't need cron. |
143 if not IsDevServer(): | 139 if not IsDevServer(): |
144 manifest_json = '/manifest.json' | 140 manifest_json = '/manifest.json' |
145 example_zips = [ | 141 example_zips = [ |
146 '%s.zip' % filename[:-len(manifest_json)] | 142 '%s.zip' % filename[:-len(manifest_json)] |
147 for filename in server_instance.content_cache.GetFromFileListing( | 143 for filename in server_instance.content_cache.GetFromFileListing( |
148 svn_constants.EXAMPLES_PATH) | 144 svn_constants.EXAMPLES_PATH) |
149 if filename.endswith(manifest_json)] | 145 if filename.endswith(manifest_json)] |
150 logging.info('cron/%s: rendering %s example zips...' % ( | 146 logging.info('cron: rendering %s example zips...' % len(example_zips)) |
151 channel, len(example_zips))) | |
152 start_time = time.time() | 147 start_time = time.time() |
153 try: | 148 try: |
154 success = success and all( | 149 success = success and all( |
155 get_via_render_servlet('extensions/examples/%s' % z).status == 200 | 150 get_via_render_servlet('extensions/examples/%s' % z).status == 200 |
156 for z in example_zips) | 151 for z in example_zips) |
157 finally: | 152 finally: |
158 logging.info('cron/%s: rendering %s example zips took %s seconds' % ( | 153 logging.info('cron: rendering %s example zips took %s seconds' % ( |
159 channel, len(example_zips), time.time() - start_time)) | 154 len(example_zips), time.time() - start_time)) |
160 | 155 |
161 except DeadlineExceededError: | 156 except DeadlineExceededError: |
162 success = False | 157 success = False |
163 | 158 |
164 logging.info("cron/%s: running Redirector cron..." % channel) | 159 logging.info('cron: running Redirector cron...') |
165 server_instance.redirector.Cron() | 160 server_instance.redirector.Cron() |
166 | 161 |
167 logging.info('cron/%s: finished' % channel) | 162 logging.info('cron: finished (%s)' % ('success' if success else 'failure',)) |
168 | 163 |
169 return (Response.Ok('Success') if success else | 164 return (Response.Ok('Success') if success else |
170 Response.InternalError('Failure')) | 165 Response.InternalError('Failure')) |
171 | 166 |
172 def _GetSafeServerInstance(self): | 167 def _GetSafeServerInstance(self): |
173 '''Returns a ServerInstance with a host file system at a safe revision, | 168 '''Returns a ServerInstance with a host file system at a safe revision, |
174 meaning the last revision that the current running version of the server | 169 meaning the last revision that the current running version of the server |
175 existed. | 170 existed. |
176 ''' | 171 ''' |
177 channel = self._channel | |
178 delegate = self._delegate | 172 delegate = self._delegate |
179 server_instance_at_head = self._CreateServerInstance(channel, None) | 173 server_instance_at_head = self._CreateServerInstance(None) |
180 | 174 |
181 app_yaml_handler = AppYamlHelper( | 175 app_yaml_handler = AppYamlHelper( |
182 svn_constants.APP_YAML_PATH, | 176 svn_constants.APP_YAML_PATH, |
183 server_instance_at_head.host_file_system, | 177 server_instance_at_head.host_file_system, |
184 server_instance_at_head.object_store_creator, | 178 server_instance_at_head.object_store_creator, |
185 server_instance_at_head.host_file_system_creator, | 179 server_instance_at_head.host_file_system_creator) |
186 self._GetBranchForChannel(channel)) | |
187 | 180 |
188 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): | 181 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): |
189 # TODO(kalman): return a new ServerInstance at an explicit revision in | 182 # TODO(kalman): return a new ServerInstance at an explicit revision in |
190 # case the HEAD version changes underneath us. | 183 # case the HEAD version changes underneath us. |
191 return server_instance_at_head | 184 return server_instance_at_head |
192 | 185 |
193 # The version in app.yaml is greater than the currently running app's. | 186 # The version in app.yaml is greater than the currently running app's. |
194 # The safe version is the one before it changed. | 187 # The safe version is the one before it changed. |
195 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( | 188 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( |
196 delegate.GetAppVersion()) - 1 | 189 delegate.GetAppVersion()) - 1 |
197 | 190 |
198 logging.info('cron/%s: app version %s is out of date, safe is %s' % ( | 191 logging.info('cron: app version %s is out of date, safe is %s' % ( |
199 channel, delegate.GetAppVersion(), safe_revision)) | 192 delegate.GetAppVersion(), safe_revision)) |
200 | 193 |
201 return self._CreateServerInstance(channel, safe_revision) | 194 return self._CreateServerInstance(safe_revision) |
202 | 195 |
203 def _CreateObjectStoreCreator(self, channel): | 196 def _CreateServerInstance(self, revision): |
204 return ObjectStoreCreator(channel, start_empty=True) | 197 object_store_creator = ObjectStoreCreator(start_empty=True) |
205 | |
206 def _GetBranchForChannel(self, channel): | |
207 object_store_creator = self._CreateObjectStoreCreator(channel) | |
208 return (self._delegate.CreateBranchUtility(object_store_creator) | |
209 .GetChannelInfo(channel).branch) | |
210 | |
211 def _CreateServerInstance(self, channel, revision): | |
212 object_store_creator = self._CreateObjectStoreCreator(channel) | |
213 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) | 198 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) |
214 host_file_system_creator = self._delegate.CreateHostFileSystemCreator( | 199 host_file_system_creator = self._delegate.CreateHostFileSystemCreator( |
215 object_store_creator) | 200 object_store_creator) |
216 host_file_system = host_file_system_creator.Create( | 201 host_file_system = host_file_system_creator.Create(revision=revision) |
217 branch_utility.GetChannelInfo(channel).branch, | |
218 revision=revision) | |
219 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem( | 202 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem( |
220 object_store_creator) | 203 object_store_creator) |
221 compiled_host_fs_factory = CompiledFileSystem.Factory( | 204 compiled_host_fs_factory = CompiledFileSystem.Factory( |
222 host_file_system, | 205 host_file_system, |
223 object_store_creator) | 206 object_store_creator) |
224 return ServerInstance(channel, | 207 return ServerInstance(object_store_creator, |
225 object_store_creator, | |
226 host_file_system, | 208 host_file_system, |
227 app_samples_file_system, | 209 app_samples_file_system, |
228 '' if channel == 'stable' else '/%s' % channel, | 210 '', |
229 compiled_host_fs_factory, | 211 compiled_host_fs_factory, |
230 branch_utility, | 212 branch_utility, |
231 host_file_system_creator) | 213 host_file_system_creator) |
OLD | NEW |