Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(62)

Side by Side Diff: chrome/common/extensions/docs/server2/cron_servlet.py

Issue 491653002: Docserver: Use GitilesFileSystem instead of SubversionFileSystem (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 6 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2013 The Chromium Authors. All rights reserved. 1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import logging 5 import logging
6 import posixpath 6 import posixpath
7 import traceback 7 import traceback
8 8
9 from app_yaml_helper import AppYamlHelper 9 from app_yaml_helper import AppYamlHelper
10 from appengine_wrappers import IsDeadlineExceededError, logservice 10 from appengine_wrappers import IsDeadlineExceededError, logservice
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
89 self._delegate = delegate_for_test or CronServlet.Delegate() 89 self._delegate = delegate_for_test or CronServlet.Delegate()
90 90
91 class Delegate(object): 91 class Delegate(object):
92 '''CronServlet's runtime dependencies. Override for testing. 92 '''CronServlet's runtime dependencies. Override for testing.
93 ''' 93 '''
94 def CreateBranchUtility(self, object_store_creator): 94 def CreateBranchUtility(self, object_store_creator):
95 return BranchUtility.Create(object_store_creator) 95 return BranchUtility.Create(object_store_creator)
96 96
97 def CreateHostFileSystemProvider(self, 97 def CreateHostFileSystemProvider(self,
98 object_store_creator, 98 object_store_creator,
99 max_trunk_revision=None): 99 pinned_commit=None):
100 return HostFileSystemProvider(object_store_creator, 100 return HostFileSystemProvider(object_store_creator,
101 max_trunk_revision=max_trunk_revision) 101 pinned_commit=pinned_commit)
102 102
103 def CreateGithubFileSystemProvider(self, object_store_creator): 103 def CreateGithubFileSystemProvider(self, object_store_creator):
104 return GithubFileSystemProvider(object_store_creator) 104 return GithubFileSystemProvider(object_store_creator)
105 105
106 def CreateGCSFileSystemProvider(self, object_store_creator): 106 def CreateGCSFileSystemProvider(self, object_store_creator):
107 return CloudStorageFileSystemProvider(object_store_creator) 107 return CloudStorageFileSystemProvider(object_store_creator)
108 108
109 def GetAppVersion(self): 109 def GetAppVersion(self):
110 return GetAppVersion() 110 return GetAppVersion()
111 111
(...skipping 21 matching lines...) Expand all
133 # little wasteful, but hopefully rendering is really fast (if it isn't we 133 # little wasteful, but hopefully rendering is really fast (if it isn't we
134 # have a problem). 134 # have a problem).
135 _cronlog.info('starting') 135 _cronlog.info('starting')
136 136
137 # This is returned every time RenderServlet wants to create a new 137 # This is returned every time RenderServlet wants to create a new
138 # ServerInstance. 138 # ServerInstance.
139 # 139 #
140 # TODO(kalman): IMPORTANT. This sometimes throws an exception, breaking 140 # TODO(kalman): IMPORTANT. This sometimes throws an exception, breaking
141 # everything. Need retry logic at the fetcher level. 141 # everything. Need retry logic at the fetcher level.
142 server_instance = self._GetSafeServerInstance() 142 server_instance = self._GetSafeServerInstance()
143 trunk_fs = server_instance.host_file_system_provider.GetTrunk() 143 master_fs = server_instance.host_file_system_provider.GetMaster()
144 144
145 def render(path): 145 def render(path):
146 request = Request(path, self._request.host, self._request.headers) 146 request = Request(path, self._request.host, self._request.headers)
147 delegate = _SingletonRenderServletDelegate(server_instance) 147 delegate = _SingletonRenderServletDelegate(server_instance)
148 return RenderServlet(request, delegate).Get() 148 return RenderServlet(request, delegate).Get()
149 149
150 def request_files_in_dir(path, prefix='', strip_ext=None): 150 def request_files_in_dir(path, prefix='', strip_ext=None):
151 '''Requests every file found under |path| in this host file system, with 151 '''Requests every file found under |path| in this host file system, with
152 a request prefix of |prefix|. |strip_ext| is an optional list of file 152 a request prefix of |prefix|. |strip_ext| is an optional list of file
153 extensions that should be stripped from paths before requesting. 153 extensions that should be stripped from paths before requesting.
154 ''' 154 '''
155 def maybe_strip_ext(name): 155 def maybe_strip_ext(name):
156 if name == SITE_VERIFICATION_FILE or not strip_ext: 156 if name == SITE_VERIFICATION_FILE or not strip_ext:
157 return name 157 return name
158 base, ext = posixpath.splitext(name) 158 base, ext = posixpath.splitext(name)
159 return base if ext in strip_ext else name 159 return base if ext in strip_ext else name
160 files = [maybe_strip_ext(name) 160 files = [maybe_strip_ext(name)
161 for name, _ in CreateURLsFromPaths(trunk_fs, path, prefix)] 161 for name, _ in CreateURLsFromPaths(master_fs, path, prefix)]
162 return _RequestEachItem(path, files, render) 162 return _RequestEachItem(path, files, render)
163 163
164 results = [] 164 results = []
165 165
166 try: 166 try:
167 # Start running the hand-written Cron methods first; they can be run in 167 # Start running the hand-written Cron methods first; they can be run in
168 # parallel. They are resolved at the end. 168 # parallel. They are resolved at the end.
169 def run_cron_for_future(target): 169 def run_cron_for_future(target):
170 title = target.__class__.__name__ 170 title = target.__class__.__name__
171 future, init_timer = TimerClosure(target.Cron) 171 future, init_timer = TimerClosure(target.Cron)
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
226 # conservative error checking), so re-raise no matter what it is. 226 # conservative error checking), so re-raise no matter what it is.
227 _cronlog.error('uncaught error: %s' % traceback.format_exc()) 227 _cronlog.error('uncaught error: %s' % traceback.format_exc())
228 raise 228 raise
229 finally: 229 finally:
230 success = all(results) 230 success = all(results)
231 _cronlog.info('finished (%s)', 'success' if success else 'FAILED') 231 _cronlog.info('finished (%s)', 'success' if success else 'FAILED')
232 return (Response.Ok('Success') if success else 232 return (Response.Ok('Success') if success else
233 Response.InternalError('Failure')) 233 Response.InternalError('Failure'))
234 234
235 def _GetSafeServerInstance(self): 235 def _GetSafeServerInstance(self):
236 '''Returns a ServerInstance with a host file system at a safe revision, 236 '''Returns a ServerInstance with a host file system at a safe commit,
237 meaning the last revision that the current running version of the server 237 meaning the last commit that the current running version of the server
238 existed. 238 existed.
239 ''' 239 '''
240 delegate = self._delegate 240 delegate = self._delegate
241 241
242 # IMPORTANT: Get a ServerInstance pinned to the most recent revision, not 242 # IMPORTANT: Get a ServerInstance pinned to the most recent commit, not
243 # HEAD. These cron jobs take a while and run very frequently such that 243 # HEAD. These cron jobs take a while and run very frequently such that
244 # there is usually one running at any given time, and eventually a file 244 # there is usually one running at any given time, and eventually a file
245 # that we're dealing with will change underneath it, putting the server in 245 # that we're dealing with will change underneath it, putting the server in
246 # an undefined state. 246 # an undefined state.
247 server_instance_near_head = self._CreateServerInstance( 247 server_instance_near_head = self._CreateServerInstance(
248 self._GetMostRecentRevision()) 248 self._GetMostRecentCommit())
249 249
250 app_yaml_handler = AppYamlHelper( 250 app_yaml_handler = AppYamlHelper(
251 server_instance_near_head.object_store_creator, 251 server_instance_near_head.object_store_creator,
252 server_instance_near_head.host_file_system_provider) 252 server_instance_near_head.host_file_system_provider)
253 253
254 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): 254 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()):
255 return server_instance_near_head 255 return server_instance_near_head
256 256
257 # The version in app.yaml is greater than the currently running app's. 257 # The version in app.yaml is greater than the currently running app's.
258 # The safe version is the one before it changed. 258 # The safe version is the one before it changed.
259 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( 259 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan(
260 delegate.GetAppVersion()) - 1 260 delegate.GetAppVersion()) - 1
261 261
262 _cronlog.info('app version %s is out of date, safe is %s', 262 _cronlog.info('app version %s is out of date, safe is %s',
263 delegate.GetAppVersion(), safe_revision) 263 delegate.GetAppVersion(), safe_revision)
264 264
265 return self._CreateServerInstance(safe_revision) 265 return self._CreateServerInstance(safe_revision)
266 266
267 def _GetMostRecentRevision(self): 267 def _GetMostRecentCommit(self):
268 '''Gets the revision of the most recent patch submitted to the host file 268 '''Gets the commit of the most recent patch submitted to the host file
269 system. This is similar to HEAD but it's a concrete revision so won't 269 system. This is similar to HEAD but it's a concrete commit so won't
270 change as the cron runs. 270 change as the cron runs.
271 ''' 271 '''
272 head_fs = ( 272 head_fs = (
273 self._CreateServerInstance(None).host_file_system_provider.GetTrunk()) 273 self._CreateServerInstance(None).host_file_system_provider.GetMaster())
274 return head_fs.Stat('').version 274 return head_fs.GetCommitID().Get()
275 275
276 def _CreateServerInstance(self, revision): 276 def _CreateServerInstance(self, commit):
277 '''Creates a ServerInstance pinned to |revision|, or HEAD if None. 277 '''Creates a ServerInstance pinned to |commit|, or HEAD if None.
278 NOTE: If passed None it's likely that during the cron run patches will be 278 NOTE: If passed None it's likely that during the cron run patches will be
279 submitted at HEAD, which may change data underneath the cron run. 279 submitted at HEAD, which may change data underneath the cron run.
280 ''' 280 '''
281 object_store_creator = ObjectStoreCreator(start_empty=True) 281 object_store_creator = ObjectStoreCreator(start_empty=True)
282 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) 282 branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
283 host_file_system_provider = self._delegate.CreateHostFileSystemProvider( 283 host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
284 object_store_creator, max_trunk_revision=revision) 284 object_store_creator, pinned_commit=commit)
285 github_file_system_provider = self._delegate.CreateGithubFileSystemProvider( 285 github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
286 object_store_creator) 286 object_store_creator)
287 gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider( 287 gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider(
288 object_store_creator) 288 object_store_creator)
289 return ServerInstance(object_store_creator, 289 return ServerInstance(object_store_creator,
290 CompiledFileSystem.Factory(object_store_creator), 290 CompiledFileSystem.Factory(object_store_creator),
291 branch_utility, 291 branch_utility,
292 host_file_system_provider, 292 host_file_system_provider,
293 github_file_system_provider, 293 github_file_system_provider,
294 gcs_file_system_provider) 294 gcs_file_system_provider)
OLDNEW
« no previous file with comments | « chrome/common/extensions/docs/server2/cron.yaml ('k') | chrome/common/extensions/docs/server2/cron_servlet_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698