| OLD | NEW |
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import os | 5 import os |
| 6 | 6 |
| 7 import object_store | 7 import object_store |
| 8 import logging |
| 8 | 9 |
| 9 APPS = 'Apps' | 10 APPS = 'Apps' |
| 10 APPS_FS = 'AppsFileSystem' | 11 APPS_FS = 'AppsFileSystem' |
| 11 CRON = 'Cron' | 12 CRON = 'Cron' |
| 12 EXTENSIONS = 'Extensions' | 13 EXTENSIONS = 'Extensions' |
| 13 EXTENSIONS_FS = 'ExtensionsFileSystem' | 14 EXTENSIONS_FS = 'ExtensionsFileSystem' |
| 14 CRON_FILE_LISTING = 'Cron.FileListing' | 15 CRON_FILE_LISTING = 'Cron.FileListing' |
| 15 CRON_GITHUB_INVALIDATION = 'Cron.GithubInvalidation' | 16 CRON_GITHUB_INVALIDATION = 'Cron.GithubInvalidation' |
| 16 CRON_INVALIDATION = 'Cron.Invalidation' | 17 CRON_INVALIDATION = 'Cron.Invalidation' |
| 17 HANDLEBAR = 'Handlebar' | 18 HANDLEBAR = 'Handlebar' |
| (...skipping 22 matching lines...) Expand all Loading... |
| 40 """A class to build a CompiledFileSystem. | 41 """A class to build a CompiledFileSystem. |
| 41 """ | 42 """ |
| 42 def __init__(self, file_system, object_store): | 43 def __init__(self, file_system, object_store): |
| 43 self._file_system = file_system | 44 self._file_system = file_system |
| 44 self._object_store = object_store | 45 self._object_store = object_store |
| 45 | 46 |
| 46 def Create(self, populate_function, namespace, version=None): | 47 def Create(self, populate_function, namespace, version=None): |
| 47 """Create a CompiledFileSystem that populates the cache by calling | 48 """Create a CompiledFileSystem that populates the cache by calling |
| 48 |populate_function| with (path, data), where |data| is the data that was | 49 |populate_function| with (path, data), where |data| is the data that was |
| 49 fetched from |path|. The keys to the cache are put in the namespace | 50 fetched from |path|. The keys to the cache are put in the namespace |
| 50 specified by |namespace|, and optionally adding |version|. """ | 51 specified by |namespace|, and optionally adding |version|. |
| 52 """ |
| 51 return CompiledFileSystem(self._file_system, | 53 return CompiledFileSystem(self._file_system, |
| 52 populate_function, | 54 populate_function, |
| 53 self._object_store, | 55 self._object_store, |
| 54 namespace, | 56 namespace, |
| 55 version=version) | 57 version=version) |
| 56 | 58 |
| 57 def __init__(self, | 59 def __init__(self, |
| 58 file_system, | 60 file_system, |
| 59 populate_function, | 61 populate_function, |
| 60 object_store, | 62 object_store, |
| 61 namespace, | 63 namespace, |
| 62 version=None): | 64 version=None): |
| 63 self._file_system = file_system | 65 self._file_system = file_system |
| 64 self._populate_function = populate_function | 66 self._populate_function = populate_function |
| 65 self._object_store = object_store | 67 self._object_store = object_store |
| 66 self._namespace = 'CompiledFileSystem.' + namespace | 68 self._namespace = 'CompiledFileSystem.' + namespace |
| 67 if version is not None: | 69 if version is not None: |
| 68 self._namespace = '%s.%s' % (self._namespace, version) | 70 self._namespace = '%s.%s' % (self._namespace, version) |
| 69 | 71 |
| 70 def _MakeKey(self, key): | 72 def _MakeKey(self, key): |
| 71 return self._namespace + '.' + key | 73 return '%s.%s' % (self._namespace, key) |
| 72 | 74 |
| 73 def _RecursiveList(self, files): | 75 def _RecursiveList(self, files): |
| 74 all_files = files[:] | 76 all_files = files[:] |
| 75 dirs = {} | 77 dirs = {} |
| 76 for filename in files: | 78 for filename in files: |
| 77 if filename.endswith('/'): | 79 if filename.endswith('/'): |
| 78 all_files.remove(filename) | 80 all_files.remove(filename) |
| 79 dirs.update(self._file_system.Read([filename]).Get()) | 81 dirs.update(self._file_system.Read([filename]).Get()) |
| 80 for dir_, files in dirs.iteritems(): | 82 for dir_, files in dirs.iteritems(): |
| 81 all_files.extend(self._RecursiveList([dir_ + f for f in files])) | 83 all_files.extend(self._RecursiveList([dir_ + f for f in files])) |
| 82 return all_files | 84 return all_files |
| 83 | 85 |
| 84 def GetFromFile(self, path, binary=False): | 86 def GetFromFile(self, path, binary=False): |
| 85 """Calls |populate_function| on the contents of the file at |path|. If | 87 """Calls |populate_function| on the contents of the file at |path|. If |
| 86 |binary| is True then the file will be read as binary - but this will only | 88 |binary| is True then the file will be read as binary - but this will only |
| 87 apply for the first time the file is fetched; if already cached, |binary| | 89 apply for the first time the file is fetched; if already cached, |binary| |
| 88 will be ignored. | 90 will be ignored. |
| 89 """ | 91 """ |
| 90 version = self._file_system.Stat(path).version | 92 version = self._file_system.Stat(path).version |
| 91 cache_entry = self._object_store.Get(self._MakeKey(path), | 93 cache_entry = self._object_store.Get(self._MakeKey(path), |
| 92 object_store.FILE_SYSTEM_CACHE, | 94 object_store.FILE_SYSTEM_CACHE, |
| 93 time=0).Get() | 95 time=0).Get() |
| 94 if (cache_entry is not None) and (version == cache_entry.version): | 96 if (cache_entry is not None) and (version == cache_entry.version): |
| 95 return cache_entry._cache_data | 97 return cache_entry._cache_data |
| 96 cache_data = self._populate_function( | 98 try: |
| 97 path, | 99 cache_data = self._populate_function( |
| 98 self._file_system.ReadSingle(path, binary=binary)) | 100 path, |
| 99 self._object_store.Set(self._MakeKey(path), | 101 self._file_system.ReadSingle(path, binary=binary)) |
| 100 _CacheEntry(cache_data, version), | 102 self._object_store.Set(self._MakeKey(path), |
| 101 object_store.FILE_SYSTEM_CACHE, | 103 _CacheEntry(cache_data, version), |
| 102 time=0) | 104 object_store.FILE_SYSTEM_CACHE, |
| 103 return cache_data | 105 time=0) |
| 106 return cache_data |
| 107 except ValueError as e: |
| 108 logging.error('Could not get data for the file at %s: %s' % (path, e)) |
| 109 return None |
| 104 | 110 |
| 105 def GetFromFileListing(self, path): | 111 def GetFromFileListing(self, path): |
| 106 """Calls |populate_function| on the listing of the files at |path|. | 112 """Calls |populate_function| on the listing of the files at |path|. |
| 107 Assumes that the path given is to a directory. | 113 Assumes that the path given is to a directory. |
| 108 """ | 114 """ |
| 109 if not path.endswith('/'): | 115 if not path.endswith('/'): |
| 110 path += '/' | 116 path += '/' |
| 111 version = self._file_system.Stat(path).version | 117 version = self._file_system.Stat(path).version |
| 112 cache_entry = self._object_store.Get( | 118 cache_entry = self._object_store.Get( |
| 113 self._MakeKey(path), | 119 self._MakeKey(path), |
| 114 object_store.FILE_SYSTEM_CACHE_LISTING, | 120 object_store.FILE_SYSTEM_CACHE_LISTING, |
| 115 time=0).Get() | 121 time=0).Get() |
| 116 if (cache_entry is not None) and (version == cache_entry.version): | 122 if (cache_entry is not None) and (version == cache_entry.version): |
| 117 return cache_entry._cache_data | 123 return cache_entry._cache_data |
| 118 cache_data = self._populate_function( | 124 cache_data = self._populate_function( |
| 119 path, | 125 path, |
| 120 self._RecursiveList( | 126 self._RecursiveList( |
| 121 [path + f for f in self._file_system.ReadSingle(path)])) | 127 [path + f for f in self._file_system.ReadSingle(path)])) |
| 122 self._object_store.Set(self._MakeKey(path), | 128 self._object_store.Set(self._MakeKey(path), |
| 123 _CacheEntry(cache_data, version), | 129 _CacheEntry(cache_data, version), |
| 124 object_store.FILE_SYSTEM_CACHE_LISTING, | 130 object_store.FILE_SYSTEM_CACHE_LISTING, |
| 125 time=0) | 131 time=0) |
| 126 return cache_data | 132 return cache_data |
| OLD | NEW |