| OLD | NEW |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import json | 5 import json |
| 6 import logging | 6 import logging |
| 7 from cStringIO import StringIO | 7 from cStringIO import StringIO |
| 8 import posixpath | 8 import posixpath |
| 9 import traceback | 9 import traceback |
| 10 from zipfile import ZipFile | 10 from zipfile import ZipFile |
| (...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 169 logging.warning('Blob for %s was corrupted in blobstore!?' % repo_key) | 169 logging.warning('Blob for %s was corrupted in blobstore!?' % repo_key) |
| 170 return fetch_from_github(version) | 170 return fetch_from_github(version) |
| 171 | 171 |
| 172 return Future(value=repo_zip) | 172 return Future(value=repo_zip) |
| 173 | 173 |
| 174 def fetch_from_github(version): | 174 def fetch_from_github(version): |
| 175 '''Returns a Future which resolves to the _GithubZipFile for this repo | 175 '''Returns a Future which resolves to the _GithubZipFile for this repo |
| 176 fetched new from GitHub, then writes it to blobstore and |version| to the | 176 fetched new from GitHub, then writes it to blobstore and |version| to the |
| 177 stat caches. | 177 stat caches. |
| 178 ''' | 178 ''' |
| 179 github_future = self._fetcher.FetchAsync( | 179 def get_zip(github_zip): |
| 180 'zipball', username=username, password=password) | |
| 181 def resolve(): | |
| 182 try: | 180 try: |
| 183 blob = github_future.Get().content | 181 blob = github_zip.content |
| 184 except urlfetch.DownloadError: | 182 except urlfetch.DownloadError: |
| 185 raise FileSystemError('Failed to download repo %s file from %s' % | 183 raise FileSystemError('Failed to download repo %s file from %s' % |
| 186 (repo_key, repo_url)) | 184 (repo_key, repo_url)) |
| 187 | 185 |
| 188 repo_zip = _GithubZipFile.Create(repo_key, blob) | 186 repo_zip = _GithubZipFile.Create(repo_key, blob) |
| 189 if repo_zip is None: | 187 if repo_zip is None: |
| 190 raise FileSystemError('Blob for %s was fetched corrupted from %s' % | 188 raise FileSystemError('Blob for %s was fetched corrupted from %s' % |
| 191 (repo_key, repo_url)) | 189 (repo_key, repo_url)) |
| 192 | 190 |
| 193 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) | 191 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) |
| 194 self._up_to_date_cache.Set(repo_key, True) | 192 self._up_to_date_cache.Set(repo_key, True) |
| 195 self._stat_cache.Set(repo_key, version) | 193 self._stat_cache.Set(repo_key, version) |
| 196 return repo_zip | 194 return repo_zip |
| 197 return Future(callback=resolve) | 195 return self._fetcher.FetchAsync( |
| 196 'zipball', username=username, password=password).Then(get_zip) |
| 198 | 197 |
| 199 # To decide whether we need to re-stat, and from there whether to re-fetch, | 198 # To decide whether we need to re-stat, and from there whether to re-fetch, |
| 200 # make use of ObjectStore's start-empty configuration. If | 199 # make use of ObjectStore's start-empty configuration. If |
| 201 # |object_store_creator| is configured to start empty then our creator | 200 # |object_store_creator| is configured to start empty then our creator |
| 202 # wants to refresh (e.g. running a cron), so fetch the live stat from | 201 # wants to refresh (e.g. running a cron), so fetch the live stat from |
| 203 # GitHub. If the stat hasn't changed since last time then no reason to | 202 # GitHub. If the stat hasn't changed since last time then no reason to |
| 204 # re-fetch from GitHub, just take from blobstore. | 203 # re-fetch from GitHub, just take from blobstore. |
| 205 | 204 |
| 206 cached_version = self._stat_cache.Get(repo_key).Get() | 205 cached_version = self._stat_cache.Get(repo_key).Get() |
| 207 if self._up_to_date_cache.Get(repo_key).Get() is None: | 206 if self._up_to_date_cache.Get(repo_key).Get() is None: |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 239 | 238 |
| 240 def Refresh(self): | 239 def Refresh(self): |
| 241 return self.ReadSingle('') | 240 return self.ReadSingle('') |
| 242 | 241 |
| 243 def Read(self, paths, skip_not_found=False): | 242 def Read(self, paths, skip_not_found=False): |
| 244 '''Returns a directory mapping |paths| to the contents of the file at each | 243 '''Returns a directory mapping |paths| to the contents of the file at each |
| 245 path. If path ends with a '/', it is treated as a directory and is mapped to | 244 path. If path ends with a '/', it is treated as a directory and is mapped to |
| 246 a list of filenames in that directory. | 245 a list of filenames in that directory. |
| 247 ''' | 246 ''' |
| 248 self._EnsureRepoZip() | 247 self._EnsureRepoZip() |
| 249 def resolve(): | 248 def read(repo_zip): |
| 250 repo_zip = self._repo_zip.Get() | |
| 251 reads = {} | 249 reads = {} |
| 252 for path in paths: | 250 for path in paths: |
| 253 if path not in repo_zip.Paths(): | 251 if path not in repo_zip.Paths(): |
| 254 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) | 252 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) |
| 255 if IsDirectory(path): | 253 if IsDirectory(path): |
| 256 reads[path] = repo_zip.List(path) | 254 reads[path] = repo_zip.List(path) |
| 257 else: | 255 else: |
| 258 reads[path] = repo_zip.Read(path) | 256 reads[path] = repo_zip.Read(path) |
| 259 return reads | 257 return reads |
| 260 return Future(callback=resolve) | 258 return self._repo_zip.Then(read) |
| 261 | 259 |
| 262 def Stat(self, path): | 260 def Stat(self, path): |
| 263 '''Stats |path| returning its version as as StatInfo object. If |path| ends | 261 '''Stats |path| returning its version as as StatInfo object. If |path| ends |
| 264 with a '/', it is assumed to be a directory and the StatInfo object returned | 262 with a '/', it is assumed to be a directory and the StatInfo object returned |
| 265 includes child_versions for all paths in the directory. | 263 includes child_versions for all paths in the directory. |
| 266 | 264 |
| 267 File paths do not include the name of the zip file, which is arbitrary and | 265 File paths do not include the name of the zip file, which is arbitrary and |
| 268 useless to consumers. | 266 useless to consumers. |
| 269 | 267 |
| 270 Because the repository will only be downloaded once per server version, all | 268 Because the repository will only be downloaded once per server version, all |
| (...skipping 16 matching lines...) Expand all Loading... |
| 287 for p in repo_zip.List(path)) | 285 for p in repo_zip.List(path)) |
| 288 return stat_info | 286 return stat_info |
| 289 | 287 |
| 290 def GetIdentity(self): | 288 def GetIdentity(self): |
| 291 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) | 289 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) |
| 292 | 290 |
| 293 def __repr__(self): | 291 def __repr__(self): |
| 294 return '%s(key=%s, url=%s)' % (type(self).__name__, | 292 return '%s(key=%s, url=%s)' % (type(self).__name__, |
| 295 self._repo_key, | 293 self._repo_key, |
| 296 self._repo_url) | 294 self._repo_url) |
| OLD | NEW |