OLD | NEW |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import json | 5 import json |
6 import logging | 6 import logging |
7 from cStringIO import StringIO | 7 from cStringIO import StringIO |
8 import posixpath | 8 import posixpath |
9 import traceback | 9 import traceback |
10 from zipfile import ZipFile | 10 from zipfile import ZipFile |
(...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
169 logging.warning('Blob for %s was corrupted in blobstore!?' % repo_key) | 169 logging.warning('Blob for %s was corrupted in blobstore!?' % repo_key) |
170 return fetch_from_github(version) | 170 return fetch_from_github(version) |
171 | 171 |
172 return Future(value=repo_zip) | 172 return Future(value=repo_zip) |
173 | 173 |
174 def fetch_from_github(version): | 174 def fetch_from_github(version): |
175 '''Returns a Future which resolves to the _GithubZipFile for this repo | 175 '''Returns a Future which resolves to the _GithubZipFile for this repo |
176 fetched new from GitHub, then writes it to blobstore and |version| to the | 176 fetched new from GitHub, then writes it to blobstore and |version| to the |
177 stat caches. | 177 stat caches. |
178 ''' | 178 ''' |
179 github_future = self._fetcher.FetchAsync( | 179 def get_zip(github_zip): |
180 'zipball', username=username, password=password) | |
181 def resolve(): | |
182 try: | 180 try: |
183 blob = github_future.Get().content | 181 blob = github_zip.content |
184 except urlfetch.DownloadError: | 182 except urlfetch.DownloadError: |
185 raise FileSystemError('Failed to download repo %s file from %s' % | 183 raise FileSystemError('Failed to download repo %s file from %s' % |
186 (repo_key, repo_url)) | 184 (repo_key, repo_url)) |
187 | 185 |
188 repo_zip = _GithubZipFile.Create(repo_key, blob) | 186 repo_zip = _GithubZipFile.Create(repo_key, blob) |
189 if repo_zip is None: | 187 if repo_zip is None: |
190 raise FileSystemError('Blob for %s was fetched corrupted from %s' % | 188 raise FileSystemError('Blob for %s was fetched corrupted from %s' % |
191 (repo_key, repo_url)) | 189 (repo_key, repo_url)) |
192 | 190 |
193 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) | 191 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) |
194 self._up_to_date_cache.Set(repo_key, True) | 192 self._up_to_date_cache.Set(repo_key, True) |
195 self._stat_cache.Set(repo_key, version) | 193 self._stat_cache.Set(repo_key, version) |
196 return repo_zip | 194 return repo_zip |
197 return Future(callback=resolve) | 195 return self._fetcher.FetchAsync( |
| 196 'zipball', username=username, password=password).Then(get_zip) |
198 | 197 |
199 # To decide whether we need to re-stat, and from there whether to re-fetch, | 198 # To decide whether we need to re-stat, and from there whether to re-fetch, |
200 # make use of ObjectStore's start-empty configuration. If | 199 # make use of ObjectStore's start-empty configuration. If |
201 # |object_store_creator| is configured to start empty then our creator | 200 # |object_store_creator| is configured to start empty then our creator |
202 # wants to refresh (e.g. running a cron), so fetch the live stat from | 201 # wants to refresh (e.g. running a cron), so fetch the live stat from |
203 # GitHub. If the stat hasn't changed since last time then no reason to | 202 # GitHub. If the stat hasn't changed since last time then no reason to |
204 # re-fetch from GitHub, just take from blobstore. | 203 # re-fetch from GitHub, just take from blobstore. |
205 | 204 |
206 cached_version = self._stat_cache.Get(repo_key).Get() | 205 cached_version = self._stat_cache.Get(repo_key).Get() |
207 if self._up_to_date_cache.Get(repo_key).Get() is None: | 206 if self._up_to_date_cache.Get(repo_key).Get() is None: |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
239 | 238 |
240 def Refresh(self): | 239 def Refresh(self): |
241 return self.ReadSingle('') | 240 return self.ReadSingle('') |
242 | 241 |
243 def Read(self, paths, skip_not_found=False): | 242 def Read(self, paths, skip_not_found=False): |
244 '''Returns a directory mapping |paths| to the contents of the file at each | 243 '''Returns a directory mapping |paths| to the contents of the file at each |
245 path. If path ends with a '/', it is treated as a directory and is mapped to | 244 path. If path ends with a '/', it is treated as a directory and is mapped to |
246 a list of filenames in that directory. | 245 a list of filenames in that directory. |
247 ''' | 246 ''' |
248 self._EnsureRepoZip() | 247 self._EnsureRepoZip() |
249 def resolve(): | 248 def read(repo_zip): |
250 repo_zip = self._repo_zip.Get() | |
251 reads = {} | 249 reads = {} |
252 for path in paths: | 250 for path in paths: |
253 if path not in repo_zip.Paths(): | 251 if path not in repo_zip.Paths(): |
254 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) | 252 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) |
255 if IsDirectory(path): | 253 if IsDirectory(path): |
256 reads[path] = repo_zip.List(path) | 254 reads[path] = repo_zip.List(path) |
257 else: | 255 else: |
258 reads[path] = repo_zip.Read(path) | 256 reads[path] = repo_zip.Read(path) |
259 return reads | 257 return reads |
260 return Future(callback=resolve) | 258 return self._repo_zip.Then(read) |
261 | 259 |
262 def Stat(self, path): | 260 def Stat(self, path): |
263 '''Stats |path| returning its version as as StatInfo object. If |path| ends | 261 '''Stats |path| returning its version as as StatInfo object. If |path| ends |
264 with a '/', it is assumed to be a directory and the StatInfo object returned | 262 with a '/', it is assumed to be a directory and the StatInfo object returned |
265 includes child_versions for all paths in the directory. | 263 includes child_versions for all paths in the directory. |
266 | 264 |
267 File paths do not include the name of the zip file, which is arbitrary and | 265 File paths do not include the name of the zip file, which is arbitrary and |
268 useless to consumers. | 266 useless to consumers. |
269 | 267 |
270 Because the repository will only be downloaded once per server version, all | 268 Because the repository will only be downloaded once per server version, all |
(...skipping 16 matching lines...) Expand all Loading... |
287 for p in repo_zip.List(path)) | 285 for p in repo_zip.List(path)) |
288 return stat_info | 286 return stat_info |
289 | 287 |
290 def GetIdentity(self): | 288 def GetIdentity(self): |
291 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) | 289 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) |
292 | 290 |
293 def __repr__(self): | 291 def __repr__(self): |
294 return '%s(key=%s, url=%s)' % (type(self).__name__, | 292 return '%s(key=%s, url=%s)' % (type(self).__name__, |
295 self._repo_key, | 293 self._repo_key, |
296 self._repo_url) | 294 self._repo_url) |
OLD | NEW |