| OLD | NEW |
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import posixpath | 5 import posixpath |
| 6 import sys | 6 import sys |
| 7 | 7 |
| 8 from file_system import FileSystem, StatInfo, FileNotFoundError | 8 from file_system import FileSystem, StatInfo, FileNotFoundError |
| 9 from future import Future | 9 from future import Future |
| 10 from path_util import IsDirectory | 10 from path_util import IsDirectory |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 56 file_version = dir_stat.child_versions.get(file_path) | 56 file_version = dir_stat.child_versions.get(file_path) |
| 57 if file_version is None: | 57 if file_version is None: |
| 58 raise FileNotFoundError('No stat found for %s in %s (found %s)' % | 58 raise FileNotFoundError('No stat found for %s in %s (found %s)' % |
| 59 (path, dir_path, dir_stat.child_versions)) | 59 (path, dir_path, dir_stat.child_versions)) |
| 60 return StatInfo(file_version) | 60 return StatInfo(file_version) |
| 61 | 61 |
| 62 dir_stat = self._stat_object_store.Get(dir_path).Get() | 62 dir_stat = self._stat_object_store.Get(dir_path).Get() |
| 63 if dir_stat is not None: | 63 if dir_stat is not None: |
| 64 return Future(value=make_stat_info(dir_stat)) | 64 return Future(value=make_stat_info(dir_stat)) |
| 65 | 65 |
| 66 dir_stat_future = self._MemoizedStatAsyncFromFileSystem(dir_path) | 66 def next(dir_stat): |
| 67 def resolve(): | |
| 68 dir_stat = dir_stat_future.Get() | |
| 69 assert dir_stat is not None # should have raised a FileNotFoundError | 67 assert dir_stat is not None # should have raised a FileNotFoundError |
| 70 # We only ever need to cache the dir stat. | 68 # We only ever need to cache the dir stat. |
| 71 self._stat_object_store.Set(dir_path, dir_stat) | 69 self._stat_object_store.Set(dir_path, dir_stat) |
| 72 return make_stat_info(dir_stat) | 70 return make_stat_info(dir_stat) |
| 73 return Future(callback=resolve) | 71 return self._MemoizedStatAsyncFromFileSystem(dir_path).Then(next) |
| 74 | 72 |
| 75 @memoize | 73 @memoize |
| 76 def _MemoizedStatAsyncFromFileSystem(self, dir_path): | 74 def _MemoizedStatAsyncFromFileSystem(self, dir_path): |
| 77 '''This is a simple wrapper to memoize Futures to directory stats, since | 75 '''This is a simple wrapper to memoize Futures to directory stats, since |
| 78 StatAsync makes heavy use of it. Only cache directories so that the | 76 StatAsync makes heavy use of it. Only cache directories so that the |
| 79 memoized cache doesn't blow up. | 77 memoized cache doesn't blow up. |
| 80 ''' | 78 ''' |
| 81 assert IsDirectory(dir_path) | 79 assert IsDirectory(dir_path) |
| 82 return self._file_system.StatAsync(dir_path) | 80 return self._file_system.StatAsync(dir_path) |
| 83 | 81 |
| 84 def Read(self, paths, skip_not_found=False): | 82 def Read(self, paths, skip_not_found=False): |
| 85 '''Reads a list of files. If a file is in memcache and it is not out of | 83 '''Reads a list of files. If a file is in memcache and it is not out of |
| 86 date, it is returned. Otherwise, the file is retrieved from the file system. | 84 date, it is returned. Otherwise, the file is retrieved from the file system. |
| 87 ''' | 85 ''' |
| 88 cached_read_values = self._read_object_store.GetMulti(paths).Get() | 86 cached_read_values = self._read_object_store.GetMulti(paths).Get() |
| 89 cached_stat_values = self._stat_object_store.GetMulti(paths).Get() | 87 cached_stat_values = self._stat_object_store.GetMulti(paths).Get() |
| 90 | 88 |
| 91 # Populate a map of paths to Futures to their stat. They may have already | 89 # Populate a map of paths to Futures to their stat. They may have already |
| 92 # been cached in which case their Future will already have been constructed | 90 # been cached in which case their Future will already have been constructed |
| 93 # with a value. | 91 # with a value. |
| 94 stat_futures = {} | 92 stat_futures = {} |
| 95 | 93 |
| 96 def swallow_file_not_found_error(future): | 94 def handle(error): |
| 97 def resolve(): | 95 if isinstance(error, FileNotFoundError): |
| 98 try: return future.Get() | 96 return None |
| 99 except FileNotFoundError: return Nnone | 97 raise error |
| 100 return Future(callback=resolve) | |
| 101 | 98 |
| 102 for path in paths: | 99 for path in paths: |
| 103 stat_value = cached_stat_values.get(path) | 100 stat_value = cached_stat_values.get(path) |
| 104 if stat_value is None: | 101 if stat_value is None: |
| 105 stat_future = self.StatAsync(path) | 102 stat_future = self.StatAsync(path) |
| 106 if skip_not_found: | 103 if skip_not_found: |
| 107 stat_future = swallow_file_not_found_error(stat_future) | 104 stat_future = stat_future.Then(lambda x: x, handle) |
| 108 else: | 105 else: |
| 109 stat_future = Future(value=stat_value) | 106 stat_future = Future(value=stat_value) |
| 110 stat_futures[path] = stat_future | 107 stat_futures[path] = stat_future |
| 111 | 108 |
| 112 # Filter only the cached data which is fresh by comparing to the latest | 109 # Filter only the cached data which is fresh by comparing to the latest |
| 113 # stat. The cached read data includes the cached version. Remove it for | 110 # stat. The cached read data includes the cached version. Remove it for |
| 114 # the result returned to callers. | 111 # the result returned to callers. |
| 115 fresh_data = dict( | 112 fresh_data = dict( |
| 116 (path, data) for path, (data, version) in cached_read_values.iteritems() | 113 (path, data) for path, (data, version) in cached_read_values.iteritems() |
| 117 if stat_futures[path].Get().version == version) | 114 if stat_futures[path].Get().version == version) |
| 118 | 115 |
| 119 if len(fresh_data) == len(paths): | 116 if len(fresh_data) == len(paths): |
| 120 # Everything was cached and up-to-date. | 117 # Everything was cached and up-to-date. |
| 121 return Future(value=fresh_data) | 118 return Future(value=fresh_data) |
| 122 | 119 |
| 123 # Read in the values that were uncached or old. | 120 def next(new_results): |
| 124 read_futures = self._file_system.Read( | |
| 125 set(paths) - set(fresh_data.iterkeys()), | |
| 126 skip_not_found=skip_not_found) | |
| 127 def resolve(): | |
| 128 new_results = read_futures.Get() | |
| 129 # Update the cache. This is a path -> (data, version) mapping. | 121 # Update the cache. This is a path -> (data, version) mapping. |
| 130 self._read_object_store.SetMulti( | 122 self._read_object_store.SetMulti( |
| 131 dict((path, (new_result, stat_futures[path].Get().version)) | 123 dict((path, (new_result, stat_futures[path].Get().version)) |
| 132 for path, new_result in new_results.iteritems())) | 124 for path, new_result in new_results.iteritems())) |
| 133 new_results.update(fresh_data) | 125 new_results.update(fresh_data) |
| 134 return new_results | 126 return new_results |
| 135 return Future(callback=resolve) | 127 # Read in the values that were uncached or old. |
| 128 return self._file_system.Read(set(paths) - set(fresh_data.iterkeys()), |
| 129 skip_not_found=skip_not_found).Then(next) |
| 136 | 130 |
| 137 def GetIdentity(self): | 131 def GetIdentity(self): |
| 138 return self._file_system.GetIdentity() | 132 return self._file_system.GetIdentity() |
| 139 | 133 |
| 140 def __repr__(self): | 134 def __repr__(self): |
| 141 return '%s of <%s>' % (type(self).__name__, repr(self._file_system)) | 135 return '%s of <%s>' % (type(self).__name__, repr(self._file_system)) |
| OLD | NEW |