OLD | NEW |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import logging | 5 import logging |
6 import posixpath | 6 import posixpath |
7 import traceback | 7 import traceback |
8 | 8 |
9 from app_yaml_helper import AppYamlHelper | 9 from app_yaml_helper import AppYamlHelper |
10 from appengine_wrappers import IsDeadlineExceededError, logservice | 10 from appengine_wrappers import IsDeadlineExceededError, logservice |
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
89 self._delegate = delegate_for_test or CronServlet.Delegate() | 89 self._delegate = delegate_for_test or CronServlet.Delegate() |
90 | 90 |
91 class Delegate(object): | 91 class Delegate(object): |
92 '''CronServlet's runtime dependencies. Override for testing. | 92 '''CronServlet's runtime dependencies. Override for testing. |
93 ''' | 93 ''' |
94 def CreateBranchUtility(self, object_store_creator): | 94 def CreateBranchUtility(self, object_store_creator): |
95 return BranchUtility.Create(object_store_creator) | 95 return BranchUtility.Create(object_store_creator) |
96 | 96 |
97 def CreateHostFileSystemProvider(self, | 97 def CreateHostFileSystemProvider(self, |
98 object_store_creator, | 98 object_store_creator, |
99 max_trunk_revision=None): | 99 pinned_commit=None): |
100 return HostFileSystemProvider(object_store_creator, | 100 return HostFileSystemProvider(object_store_creator, |
101 max_trunk_revision=max_trunk_revision) | 101 pinned_commit=pinned_commit) |
102 | 102 |
103 def CreateGithubFileSystemProvider(self, object_store_creator): | 103 def CreateGithubFileSystemProvider(self, object_store_creator): |
104 return GithubFileSystemProvider(object_store_creator) | 104 return GithubFileSystemProvider(object_store_creator) |
105 | 105 |
106 def CreateGCSFileSystemProvider(self, object_store_creator): | 106 def CreateGCSFileSystemProvider(self, object_store_creator): |
107 return CloudStorageFileSystemProvider(object_store_creator) | 107 return CloudStorageFileSystemProvider(object_store_creator) |
108 | 108 |
109 def GetAppVersion(self): | 109 def GetAppVersion(self): |
110 return GetAppVersion() | 110 return GetAppVersion() |
111 | 111 |
(...skipping 21 matching lines...) Expand all Loading... |
133 # little wasteful, but hopefully rendering is really fast (if it isn't we | 133 # little wasteful, but hopefully rendering is really fast (if it isn't we |
134 # have a problem). | 134 # have a problem). |
135 _cronlog.info('starting') | 135 _cronlog.info('starting') |
136 | 136 |
137 # This is returned every time RenderServlet wants to create a new | 137 # This is returned every time RenderServlet wants to create a new |
138 # ServerInstance. | 138 # ServerInstance. |
139 # | 139 # |
140 # TODO(kalman): IMPORTANT. This sometimes throws an exception, breaking | 140 # TODO(kalman): IMPORTANT. This sometimes throws an exception, breaking |
141 # everything. Need retry logic at the fetcher level. | 141 # everything. Need retry logic at the fetcher level. |
142 server_instance = self._GetSafeServerInstance() | 142 server_instance = self._GetSafeServerInstance() |
143 trunk_fs = server_instance.host_file_system_provider.GetTrunk() | 143 master_fs = server_instance.host_file_system_provider.GetMaster() |
144 | 144 |
145 def render(path): | 145 def render(path): |
146 request = Request(path, self._request.host, self._request.headers) | 146 request = Request(path, self._request.host, self._request.headers) |
147 delegate = _SingletonRenderServletDelegate(server_instance) | 147 delegate = _SingletonRenderServletDelegate(server_instance) |
148 return RenderServlet(request, delegate).Get() | 148 return RenderServlet(request, delegate).Get() |
149 | 149 |
150 def request_files_in_dir(path, prefix='', strip_ext=None): | 150 def request_files_in_dir(path, prefix='', strip_ext=None): |
151 '''Requests every file found under |path| in this host file system, with | 151 '''Requests every file found under |path| in this host file system, with |
152 a request prefix of |prefix|. |strip_ext| is an optional list of file | 152 a request prefix of |prefix|. |strip_ext| is an optional list of file |
153 extensions that should be stripped from paths before requesting. | 153 extensions that should be stripped from paths before requesting. |
154 ''' | 154 ''' |
155 def maybe_strip_ext(name): | 155 def maybe_strip_ext(name): |
156 if name == SITE_VERIFICATION_FILE or not strip_ext: | 156 if name == SITE_VERIFICATION_FILE or not strip_ext: |
157 return name | 157 return name |
158 base, ext = posixpath.splitext(name) | 158 base, ext = posixpath.splitext(name) |
159 return base if ext in strip_ext else name | 159 return base if ext in strip_ext else name |
160 files = [maybe_strip_ext(name) | 160 files = [maybe_strip_ext(name) |
161 for name, _ in CreateURLsFromPaths(trunk_fs, path, prefix)] | 161 for name, _ in CreateURLsFromPaths(master_fs, path, prefix)] |
162 return _RequestEachItem(path, files, render) | 162 return _RequestEachItem(path, files, render) |
163 | 163 |
164 results = [] | 164 results = [] |
165 | 165 |
166 try: | 166 try: |
167 # Start running the hand-written Cron methods first; they can be run in | 167 # Start running the hand-written Cron methods first; they can be run in |
168 # parallel. They are resolved at the end. | 168 # parallel. They are resolved at the end. |
169 def run_cron_for_future(target): | 169 def run_cron_for_future(target): |
170 title = target.__class__.__name__ | 170 title = target.__class__.__name__ |
171 future, init_timer = TimerClosure(target.Cron) | 171 future, init_timer = TimerClosure(target.Cron) |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
226 # conservative error checking), so re-raise no matter what it is. | 226 # conservative error checking), so re-raise no matter what it is. |
227 _cronlog.error('uncaught error: %s' % traceback.format_exc()) | 227 _cronlog.error('uncaught error: %s' % traceback.format_exc()) |
228 raise | 228 raise |
229 finally: | 229 finally: |
230 success = all(results) | 230 success = all(results) |
231 _cronlog.info('finished (%s)', 'success' if success else 'FAILED') | 231 _cronlog.info('finished (%s)', 'success' if success else 'FAILED') |
232 return (Response.Ok('Success') if success else | 232 return (Response.Ok('Success') if success else |
233 Response.InternalError('Failure')) | 233 Response.InternalError('Failure')) |
234 | 234 |
235 def _GetSafeServerInstance(self): | 235 def _GetSafeServerInstance(self): |
236 '''Returns a ServerInstance with a host file system at a safe revision, | 236 '''Returns a ServerInstance with a host file system at a safe commit, |
237 meaning the last revision that the current running version of the server | 237 meaning the last commit that the current running version of the server |
238 existed. | 238 existed. |
239 ''' | 239 ''' |
240 delegate = self._delegate | 240 delegate = self._delegate |
241 | 241 |
242 # IMPORTANT: Get a ServerInstance pinned to the most recent revision, not | 242 # IMPORTANT: Get a ServerInstance pinned to the most recent commit, not |
243 # HEAD. These cron jobs take a while and run very frequently such that | 243 # HEAD. These cron jobs take a while and run very frequently such that |
244 # there is usually one running at any given time, and eventually a file | 244 # there is usually one running at any given time, and eventually a file |
245 # that we're dealing with will change underneath it, putting the server in | 245 # that we're dealing with will change underneath it, putting the server in |
246 # an undefined state. | 246 # an undefined state. |
247 server_instance_near_head = self._CreateServerInstance( | 247 server_instance_near_head = self._CreateServerInstance( |
248 self._GetMostRecentRevision()) | 248 self._GetMostRecentCommit()) |
249 | 249 |
250 app_yaml_handler = AppYamlHelper( | 250 app_yaml_handler = AppYamlHelper( |
251 server_instance_near_head.object_store_creator, | 251 server_instance_near_head.object_store_creator, |
252 server_instance_near_head.host_file_system_provider) | 252 server_instance_near_head.host_file_system_provider) |
253 | 253 |
254 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): | 254 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): |
255 return server_instance_near_head | 255 return server_instance_near_head |
256 | 256 |
257 # The version in app.yaml is greater than the currently running app's. | 257 # The version in app.yaml is greater than the currently running app's. |
258 # The safe version is the one before it changed. | 258 # The safe version is the one before it changed. |
259 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( | 259 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( |
260 delegate.GetAppVersion()) - 1 | 260 delegate.GetAppVersion()) - 1 |
261 | 261 |
262 _cronlog.info('app version %s is out of date, safe is %s', | 262 _cronlog.info('app version %s is out of date, safe is %s', |
263 delegate.GetAppVersion(), safe_revision) | 263 delegate.GetAppVersion(), safe_revision) |
264 | 264 |
265 return self._CreateServerInstance(safe_revision) | 265 return self._CreateServerInstance(safe_revision) |
266 | 266 |
267 def _GetMostRecentRevision(self): | 267 def _GetMostRecentCommit(self): |
268 '''Gets the revision of the most recent patch submitted to the host file | 268 '''Gets the commit of the most recent patch submitted to the host file |
269 system. This is similar to HEAD but it's a concrete revision so won't | 269 system. This is similar to HEAD but it's a concrete commit so won't |
270 change as the cron runs. | 270 change as the cron runs. |
271 ''' | 271 ''' |
272 head_fs = ( | 272 head_fs = ( |
273 self._CreateServerInstance(None).host_file_system_provider.GetTrunk()) | 273 self._CreateServerInstance(None).host_file_system_provider.GetMaster()) |
274 return head_fs.Stat('').version | 274 return head_fs.GetCommitID().Get() |
275 | 275 |
276 def _CreateServerInstance(self, revision): | 276 def _CreateServerInstance(self, commit): |
277 '''Creates a ServerInstance pinned to |revision|, or HEAD if None. | 277 '''Creates a ServerInstance pinned to |commit|, or HEAD if None. |
278 NOTE: If passed None it's likely that during the cron run patches will be | 278 NOTE: If passed None it's likely that during the cron run patches will be |
279 submitted at HEAD, which may change data underneath the cron run. | 279 submitted at HEAD, which may change data underneath the cron run. |
280 ''' | 280 ''' |
281 object_store_creator = ObjectStoreCreator(start_empty=True) | 281 object_store_creator = ObjectStoreCreator(start_empty=True) |
282 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) | 282 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) |
283 host_file_system_provider = self._delegate.CreateHostFileSystemProvider( | 283 host_file_system_provider = self._delegate.CreateHostFileSystemProvider( |
284 object_store_creator, max_trunk_revision=revision) | 284 object_store_creator, pinned_commit=commit) |
285 github_file_system_provider = self._delegate.CreateGithubFileSystemProvider( | 285 github_file_system_provider = self._delegate.CreateGithubFileSystemProvider( |
286 object_store_creator) | 286 object_store_creator) |
287 gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider( | 287 gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider( |
288 object_store_creator) | 288 object_store_creator) |
289 return ServerInstance(object_store_creator, | 289 return ServerInstance(object_store_creator, |
290 CompiledFileSystem.Factory(object_store_creator), | 290 CompiledFileSystem.Factory(object_store_creator), |
291 branch_utility, | 291 branch_utility, |
292 host_file_system_provider, | 292 host_file_system_provider, |
293 github_file_system_provider, | 293 github_file_system_provider, |
294 gcs_file_system_provider) | 294 gcs_file_system_provider) |
OLD | NEW |