OLD | NEW |
---|---|
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import json | 5 import json |
6 import logging | 6 import logging |
7 from cStringIO import StringIO | 7 from cStringIO import StringIO |
8 import posixpath | 8 import posixpath |
9 import traceback | 9 import traceback |
10 from zipfile import ZipFile | 10 from zipfile import ZipFile |
(...skipping 160 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
171 | 171 |
172 return Future(value=repo_zip) | 172 return Future(value=repo_zip) |
173 | 173 |
174 def fetch_from_github(version): | 174 def fetch_from_github(version): |
175 '''Returns a Future which resolves to the _GithubZipFile for this repo | 175 '''Returns a Future which resolves to the _GithubZipFile for this repo |
176 fetched new from GitHub, then writes it to blobstore and |version| to the | 176 fetched new from GitHub, then writes it to blobstore and |version| to the |
177 stat caches. | 177 stat caches. |
178 ''' | 178 ''' |
179 github_future = self._fetcher.FetchAsync( | 179 github_future = self._fetcher.FetchAsync( |
180 'zipball', username=username, password=password) | 180 'zipball', username=username, password=password) |
181 def resolve(): | 181 def resolve(github_zip): |
ahernandez
2014/09/03 19:16:07
Nit: more descriptive name, e.g. get_zip.
| |
182 try: | 182 try: |
183 blob = github_future.Get().content | 183 blob = github_zip.content |
184 except urlfetch.DownloadError: | 184 except urlfetch.DownloadError: |
185 raise FileSystemError('Failed to download repo %s file from %s' % | 185 raise FileSystemError('Failed to download repo %s file from %s' % |
186 (repo_key, repo_url)) | 186 (repo_key, repo_url)) |
187 | 187 |
188 repo_zip = _GithubZipFile.Create(repo_key, blob) | 188 repo_zip = _GithubZipFile.Create(repo_key, blob) |
189 if repo_zip is None: | 189 if repo_zip is None: |
190 raise FileSystemError('Blob for %s was fetched corrupted from %s' % | 190 raise FileSystemError('Blob for %s was fetched corrupted from %s' % |
191 (repo_key, repo_url)) | 191 (repo_key, repo_url)) |
192 | 192 |
193 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) | 193 self._blobstore.Set(self._repo_url, blob, _GITHUB_REPOS_NAMESPACE) |
194 self._up_to_date_cache.Set(repo_key, True) | 194 self._up_to_date_cache.Set(repo_key, True) |
195 self._stat_cache.Set(repo_key, version) | 195 self._stat_cache.Set(repo_key, version) |
196 return repo_zip | 196 return repo_zip |
197 return Future(callback=resolve) | 197 return github_future.Then(resolve) |
ahernandez
2014/09/03 19:12:37
Inline github_future.
| |
198 | 198 |
199 # To decide whether we need to re-stat, and from there whether to re-fetch, | 199 # To decide whether we need to re-stat, and from there whether to re-fetch, |
200 # make use of ObjectStore's start-empty configuration. If | 200 # make use of ObjectStore's start-empty configuration. If |
201 # |object_store_creator| is configured to start empty then our creator | 201 # |object_store_creator| is configured to start empty then our creator |
202 # wants to refresh (e.g. running a cron), so fetch the live stat from | 202 # wants to refresh (e.g. running a cron), so fetch the live stat from |
203 # GitHub. If the stat hasn't changed since last time then no reason to | 203 # GitHub. If the stat hasn't changed since last time then no reason to |
204 # re-fetch from GitHub, just take from blobstore. | 204 # re-fetch from GitHub, just take from blobstore. |
205 | 205 |
206 cached_version = self._stat_cache.Get(repo_key).Get() | 206 cached_version = self._stat_cache.Get(repo_key).Get() |
207 if self._up_to_date_cache.Get(repo_key).Get() is None: | 207 if self._up_to_date_cache.Get(repo_key).Get() is None: |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
239 | 239 |
240 def Refresh(self): | 240 def Refresh(self): |
241 return self.ReadSingle('') | 241 return self.ReadSingle('') |
242 | 242 |
243 def Read(self, paths, skip_not_found=False): | 243 def Read(self, paths, skip_not_found=False): |
244 '''Returns a directory mapping |paths| to the contents of the file at each | 244 '''Returns a directory mapping |paths| to the contents of the file at each |
245 path. If path ends with a '/', it is treated as a directory and is mapped to | 245 path. If path ends with a '/', it is treated as a directory and is mapped to |
246 a list of filenames in that directory. | 246 a list of filenames in that directory. |
247 ''' | 247 ''' |
248 self._EnsureRepoZip() | 248 self._EnsureRepoZip() |
249 def resolve(): | 249 def resolve(repo_zip): |
ahernandez
2014/09/03 19:16:07
Nit: choose a more descriptive name, e.g. read.
| |
250 repo_zip = self._repo_zip.Get() | |
251 reads = {} | 250 reads = {} |
252 for path in paths: | 251 for path in paths: |
253 if path not in repo_zip.Paths(): | 252 if path not in repo_zip.Paths(): |
254 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) | 253 raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) |
255 if IsDirectory(path): | 254 if IsDirectory(path): |
256 reads[path] = repo_zip.List(path) | 255 reads[path] = repo_zip.List(path) |
257 else: | 256 else: |
258 reads[path] = repo_zip.Read(path) | 257 reads[path] = repo_zip.Read(path) |
259 return reads | 258 return reads |
260 return Future(callback=resolve) | 259 return self._repo_zip.Then(resolve) |
261 | 260 |
262 def Stat(self, path): | 261 def Stat(self, path): |
263 '''Stats |path| returning its version as as StatInfo object. If |path| ends | 262 '''Stats |path| returning its version as as StatInfo object. If |path| ends |
264 with a '/', it is assumed to be a directory and the StatInfo object returned | 263 with a '/', it is assumed to be a directory and the StatInfo object returned |
265 includes child_versions for all paths in the directory. | 264 includes child_versions for all paths in the directory. |
266 | 265 |
267 File paths do not include the name of the zip file, which is arbitrary and | 266 File paths do not include the name of the zip file, which is arbitrary and |
268 useless to consumers. | 267 useless to consumers. |
269 | 268 |
270 Because the repository will only be downloaded once per server version, all | 269 Because the repository will only be downloaded once per server version, all |
(...skipping 16 matching lines...) Expand all Loading... | |
287 for p in repo_zip.List(path)) | 286 for p in repo_zip.List(path)) |
288 return stat_info | 287 return stat_info |
289 | 288 |
290 def GetIdentity(self): | 289 def GetIdentity(self): |
291 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) | 290 return '%s' % StringIdentity(self.__class__.__name__ + self._repo_key) |
292 | 291 |
293 def __repr__(self): | 292 def __repr__(self): |
294 return '%s(key=%s, url=%s)' % (type(self).__name__, | 293 return '%s(key=%s, url=%s)' % (type(self).__name__, |
295 self._repo_key, | 294 self._repo_key, |
296 self._repo_url) | 295 self._repo_url) |
OLD | NEW |