Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(586)

Side by Side Diff: chrome/common/extensions/docs/server2/new_github_file_system.py

Issue 532903002: Docserver: Convert more future Get calls to Thens (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Nits Created 6 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2013 The Chromium Authors. All rights reserved. 1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import json 5 import json
6 import logging 6 import logging
7 from cStringIO import StringIO 7 from cStringIO import StringIO
8 import posixpath 8 import posixpath
9 import traceback 9 import traceback
10 from zipfile import ZipFile 10 from zipfile import ZipFile
(...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after
169 logging.warning('Blob for %s was corrupted in blobstore!?' % repo_key) 169 logging.warning('Blob for %s was corrupted in blobstore!?' % repo_key)
170 return fetch_from_github(version) 170 return fetch_from_github(version)
171 171
172 return Future(value=repo_zip) 172 return Future(value=repo_zip)
173 173
174 def fetch_from_github(version): 174 def fetch_from_github(version):
175 '''Returns a Future which resolves to the _GithubZipFile for this repo 175 '''Returns a Future which resolves to the _GithubZipFile for this repo
176 fetched new from GitHub, then writes it to blobstore and |version| to the 176 fetched new from GitHub, then writes it to blobstore and |version| to the
177 stat caches. 177 stat caches.
178 ''' 178 '''
179 github_future = self._fetcher.FetchAsync( 179 def get_zip(github_zip):
180 'zipball', username=username, password=password)
181 def resolve():
182 try: 180 try:
183 blob = github_future.Get().content 181 blob = github_zip.content
184 except urlfetch.DownloadError: 182 except urlfetch.DownloadError:
185 raise FileSystemError('Failed to download repo %s file from %s' % 183 raise FileSystemError('Failed to download repo %s file from %s' %
186 (repo_key, repo_url)) 184 (repo_key, repo_url))
187 185
188 repo_zip = _GithubZipFile.Create(repo_key, blob) 186 repo_zip = _GithubZipFile.Create(repo_key, blob)
189 if repo_zip is None: 187 if repo_zip is None:
190 raise FileSystemError('Blob for %s was fetched corrupted from %s' % 188 raise FileSystemError('Blob for %s was fetched corrupted from %s' %
191 (repo_key, repo_url)) 189 (repo_key, repo_url))
192 190
193 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) 191 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE)
194 self._up_to_date_cache.Set(repo_key, True) 192 self._up_to_date_cache.Set(repo_key, True)
195 self._stat_cache.Set(repo_key, version) 193 self._stat_cache.Set(repo_key, version)
196 return repo_zip 194 return repo_zip
197 return Future(callback=resolve) 195 return self._fetcher.FetchAsync(
196 'zipball', username=username, password=password).Then(get_zip)
198 197
199 # To decide whether we need to re-stat, and from there whether to re-fetch, 198 # To decide whether we need to re-stat, and from there whether to re-fetch,
200 # make use of ObjectStore's start-empty configuration. If 199 # make use of ObjectStore's start-empty configuration. If
201 # |object_store_creator| is configured to start empty then our creator 200 # |object_store_creator| is configured to start empty then our creator
202 # wants to refresh (e.g. running a cron), so fetch the live stat from 201 # wants to refresh (e.g. running a cron), so fetch the live stat from
203 # GitHub. If the stat hasn't changed since last time then no reason to 202 # GitHub. If the stat hasn't changed since last time then no reason to
204 # re-fetch from GitHub, just take from blobstore. 203 # re-fetch from GitHub, just take from blobstore.
205 204
206 cached_version = self._stat_cache.Get(repo_key).Get() 205 cached_version = self._stat_cache.Get(repo_key).Get()
207 if self._up_to_date_cache.Get(repo_key).Get() is None: 206 if self._up_to_date_cache.Get(repo_key).Get() is None:
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
239 238
240 def Refresh(self): 239 def Refresh(self):
241 return self.ReadSingle('') 240 return self.ReadSingle('')
242 241
243 def Read(self, paths, skip_not_found=False): 242 def Read(self, paths, skip_not_found=False):
244 '''Returns a directory mapping |paths| to the contents of the file at each 243 '''Returns a directory mapping |paths| to the contents of the file at each
245 path. If path ends with a '/', it is treated as a directory and is mapped to 244 path. If path ends with a '/', it is treated as a directory and is mapped to
246 a list of filenames in that directory. 245 a list of filenames in that directory.
247 ''' 246 '''
248 self._EnsureRepoZip() 247 self._EnsureRepoZip()
249 def resolve(): 248 def read(repo_zip):
250 repo_zip = self._repo_zip.Get()
251 reads = {} 249 reads = {}
252 for path in paths: 250 for path in paths:
253 if path not in repo_zip.Paths(): 251 if path not in repo_zip.Paths():
254 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) 252 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path))
255 if IsDirectory(path): 253 if IsDirectory(path):
256 reads[path] = repo_zip.List(path) 254 reads[path] = repo_zip.List(path)
257 else: 255 else:
258 reads[path] = repo_zip.Read(path) 256 reads[path] = repo_zip.Read(path)
259 return reads 257 return reads
260 return Future(callback=resolve) 258 return self._repo_zip.Then(read)
261 259
262 def Stat(self, path): 260 def Stat(self, path):
263 '''Stats |path| returning its version as as StatInfo object. If |path| ends 261 '''Stats |path| returning its version as as StatInfo object. If |path| ends
264 with a '/', it is assumed to be a directory and the StatInfo object returned 262 with a '/', it is assumed to be a directory and the StatInfo object returned
265 includes child_versions for all paths in the directory. 263 includes child_versions for all paths in the directory.
266 264
267 File paths do not include the name of the zip file, which is arbitrary and 265 File paths do not include the name of the zip file, which is arbitrary and
268 useless to consumers. 266 useless to consumers.
269 267
270 Because the repository will only be downloaded once per server version, all 268 Because the repository will only be downloaded once per server version, all
(...skipping 16 matching lines...) Expand all
287 for p in repo_zip.List(path)) 285 for p in repo_zip.List(path))
288 return stat_info 286 return stat_info
289 287
290 def GetIdentity(self): 288 def GetIdentity(self):
291 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) 289 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key)
292 290
293 def __repr__(self): 291 def __repr__(self):
294 return '%s(key=%s, url=%s)' % (type(self).__name__, 292 return '%s(key=%s, url=%s)' % (type(self).__name__,
295 self._repo_key, 293 self._repo_key,
296 self._repo_url) 294 self._repo_url)
OLDNEW
« no previous file with comments | « chrome/common/extensions/docs/server2/features_bundle.py ('k') | chrome/common/extensions/docs/server2/path_canonicalizer.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698