Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import json | 5 import json |
| 6 import logging | 6 import logging |
| 7 from cStringIO import StringIO | 7 from cStringIO import StringIO |
| 8 import posixpath | 8 import posixpath |
| 9 import traceback | 9 import traceback |
| 10 from zipfile import ZipFile | 10 from zipfile import ZipFile |
| (...skipping 160 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 171 | 171 |
| 172 return Future(value=repo_zip) | 172 return Future(value=repo_zip) |
| 173 | 173 |
| 174 def fetch_from_github(version): | 174 def fetch_from_github(version): |
| 175 '''Returns a Future which resolves to the _GithubZipFile for this repo | 175 '''Returns a Future which resolves to the _GithubZipFile for this repo |
| 176 fetched new from GitHub, then writes it to blobstore and |version| to the | 176 fetched new from GitHub, then writes it to blobstore and |version| to the |
| 177 stat caches. | 177 stat caches. |
| 178 ''' | 178 ''' |
| 179 github_future = self._fetcher.FetchAsync( | 179 github_future = self._fetcher.FetchAsync( |
| 180 'zipball', username=username, password=password) | 180 'zipball', username=username, password=password) |
| 181 def resolve(): | 181 def resolve(github_zip): |
|
ahernandez
2014/09/03 19:16:07
Nit: more descriptive name, e.g. get_zip.
| |
| 182 try: | 182 try: |
| 183 blob = github_future.Get().content | 183 blob = github_zip.content |
| 184 except urlfetch.DownloadError: | 184 except urlfetch.DownloadError: |
| 185 raise FileSystemError('Failed to download repo %s file from %s' % | 185 raise FileSystemError('Failed to download repo %s file from %s' % |
| 186 (repo_key, repo_url)) | 186 (repo_key, repo_url)) |
| 187 | 187 |
| 188 repo_zip = _GithubZipFile.Create(repo_key, blob) | 188 repo_zip = _GithubZipFile.Create(repo_key, blob) |
| 189 if repo_zip is None: | 189 if repo_zip is None: |
| 190 raise FileSystemError('Blob for %s was fetched corrupted from %s' % | 190 raise FileSystemError('Blob for %s was fetched corrupted from %s' % |
| 191 (repo_key, repo_url)) | 191 (repo_key, repo_url)) |
| 192 | 192 |
| 193 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) | 193 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) |
| 194 self._up_to_date_cache.Set(repo_key, True) | 194 self._up_to_date_cache.Set(repo_key, True) |
| 195 self._stat_cache.Set(repo_key, version) | 195 self._stat_cache.Set(repo_key, version) |
| 196 return repo_zip | 196 return repo_zip |
| 197 return Future(callback=resolve) | 197 return github_future.Then(resolve) |
|
ahernandez
2014/09/03 19:12:37
Inline github_future.
| |
| 198 | 198 |
| 199 # To decide whether we need to re-stat, and from there whether to re-fetch, | 199 # To decide whether we need to re-stat, and from there whether to re-fetch, |
| 200 # make use of ObjectStore's start-empty configuration. If | 200 # make use of ObjectStore's start-empty configuration. If |
| 201 # |object_store_creator| is configured to start empty then our creator | 201 # |object_store_creator| is configured to start empty then our creator |
| 202 # wants to refresh (e.g. running a cron), so fetch the live stat from | 202 # wants to refresh (e.g. running a cron), so fetch the live stat from |
| 203 # GitHub. If the stat hasn't changed since last time then no reason to | 203 # GitHub. If the stat hasn't changed since last time then no reason to |
| 204 # re-fetch from GitHub, just take from blobstore. | 204 # re-fetch from GitHub, just take from blobstore. |
| 205 | 205 |
| 206 cached_version = self._stat_cache.Get(repo_key).Get() | 206 cached_version = self._stat_cache.Get(repo_key).Get() |
| 207 if self._up_to_date_cache.Get(repo_key).Get() is None: | 207 if self._up_to_date_cache.Get(repo_key).Get() is None: |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 239 | 239 |
| 240 def Refresh(self): | 240 def Refresh(self): |
| 241 return self.ReadSingle('') | 241 return self.ReadSingle('') |
| 242 | 242 |
| 243 def Read(self, paths, skip_not_found=False): | 243 def Read(self, paths, skip_not_found=False): |
| 244 '''Returns a directory mapping |paths| to the contents of the file at each | 244 '''Returns a directory mapping |paths| to the contents of the file at each |
| 245 path. If path ends with a '/', it is treated as a directory and is mapped to | 245 path. If path ends with a '/', it is treated as a directory and is mapped to |
| 246 a list of filenames in that directory. | 246 a list of filenames in that directory. |
| 247 ''' | 247 ''' |
| 248 self._EnsureRepoZip() | 248 self._EnsureRepoZip() |
| 249 def resolve(): | 249 def resolve(repo_zip): |
|
ahernandez
2014/09/03 19:16:07
Nit: choose a more descriptive name, e.g. read.
| |
| 250 repo_zip = self._repo_zip.Get() | |
| 251 reads = {} | 250 reads = {} |
| 252 for path in paths: | 251 for path in paths: |
| 253 if path not in repo_zip.Paths(): | 252 if path not in repo_zip.Paths(): |
| 254 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) | 253 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) |
| 255 if IsDirectory(path): | 254 if IsDirectory(path): |
| 256 reads[path] = repo_zip.List(path) | 255 reads[path] = repo_zip.List(path) |
| 257 else: | 256 else: |
| 258 reads[path] = repo_zip.Read(path) | 257 reads[path] = repo_zip.Read(path) |
| 259 return reads | 258 return reads |
| 260 return Future(callback=resolve) | 259 return self._repo_zip.Then(resolve) |
| 261 | 260 |
| 262 def Stat(self, path): | 261 def Stat(self, path): |
| 263 '''Stats |path| returning its version as as StatInfo object. If |path| ends | 262 '''Stats |path| returning its version as as StatInfo object. If |path| ends |
| 264 with a '/', it is assumed to be a directory and the StatInfo object returned | 263 with a '/', it is assumed to be a directory and the StatInfo object returned |
| 265 includes child_versions for all paths in the directory. | 264 includes child_versions for all paths in the directory. |
| 266 | 265 |
| 267 File paths do not include the name of the zip file, which is arbitrary and | 266 File paths do not include the name of the zip file, which is arbitrary and |
| 268 useless to consumers. | 267 useless to consumers. |
| 269 | 268 |
| 270 Because the repository will only be downloaded once per server version, all | 269 Because the repository will only be downloaded once per server version, all |
| (...skipping 16 matching lines...) Expand all Loading... | |
| 287 for p in repo_zip.List(path)) | 286 for p in repo_zip.List(path)) |
| 288 return stat_info | 287 return stat_info |
| 289 | 288 |
| 290 def GetIdentity(self): | 289 def GetIdentity(self): |
| 291 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) | 290 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) |
| 292 | 291 |
| 293 def __repr__(self): | 292 def __repr__(self): |
| 294 return '%s(key=%s, url=%s)' % (type(self).__name__, | 293 return '%s(key=%s, url=%s)' % (type(self).__name__, |
| 295 self._repo_key, | 294 self._repo_key, |
| 296 self._repo_url) | 295 self._repo_url) |
| OLD | NEW |