Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import os | 5 import os |
| 6 | 6 |
| 7 import object_store | 7 import object_store |
| 8 import logging | |
| 8 | 9 |
| 9 APPS = 'Apps' | 10 APPS = 'Apps' |
| 10 APPS_FS = 'AppsFileSystem' | 11 APPS_FS = 'AppsFileSystem' |
| 11 CRON = 'Cron' | 12 CRON = 'Cron' |
| 12 EXTENSIONS = 'Extensions' | 13 EXTENSIONS = 'Extensions' |
| 13 EXTENSIONS_FS = 'ExtensionsFileSystem' | 14 EXTENSIONS_FS = 'ExtensionsFileSystem' |
| 14 CRON_FILE_LISTING = 'Cron.FileListing' | 15 CRON_FILE_LISTING = 'Cron.FileListing' |
| 15 CRON_GITHUB_INVALIDATION = 'Cron.GithubInvalidation' | 16 CRON_GITHUB_INVALIDATION = 'Cron.GithubInvalidation' |
| 16 CRON_INVALIDATION = 'Cron.Invalidation' | 17 CRON_INVALIDATION = 'Cron.Invalidation' |
| 17 HANDLEBAR = 'Handlebar' | 18 HANDLEBAR = 'Handlebar' |
| 18 IDL = 'IDL' | 19 IDL = 'IDL' |
| 19 IDL_NO_REFS = 'IDLNoRefs' | 20 IDL_NO_REFS = 'IDLNoRefs' |
| 20 IDL_NAMES = 'IDLNames' | 21 IDL_NAMES = 'IDLNames' |
| 21 INTRO = 'Intro' | 22 INTRO = 'Intro' |
| 22 JSON = 'JSON' | 23 JSON = 'JSON' |
| 23 JSON_NO_REFS = 'JSONNoRefs' | 24 JSON_NO_REFS = 'JSONNoRefs' |
| 24 LIST = 'List' | 25 LIST = 'List' |
| 25 NAMES = 'Names' | 26 NAMES = 'Names' |
| 26 PERMS = 'Perms' | 27 PERMS = 'Perms' |
| 27 SIDENAV = 'Sidenav' | 28 SIDENAV = 'Sidenav' |
| 28 STATIC = 'Static' | 29 STATIC = 'Static' |
| 29 ZIP = 'Zip' | 30 ZIP = 'Zip' |
| 30 | 31 DEFAULT_BRANCH = '000' |
|
cduvall
2013/03/21 18:43:53
Separate this from the namespace constants.
epeterson
2013/03/25 19:35:11
Done.
| |
| 31 class _CacheEntry(object): | 32 class _CacheEntry(object): |
| 32 def __init__(self, cache_data, version): | 33 def __init__(self, cache_data, version): |
| 33 self._cache_data = cache_data | 34 self._cache_data = cache_data |
| 34 self.version = version | 35 self.version = version |
| 35 | 36 |
| 36 class CompiledFileSystem(object): | 37 class CompiledFileSystem(object): |
| 37 """This class caches FileSystem data that has been processed. | 38 """This class caches FileSystem data that has been processed. |
| 38 """ | 39 """ |
| 39 class Factory(object): | 40 class Factory(object): |
| 40 """A class to build a CompiledFileSystem. | 41 """A class to build a CompiledFileSystem. |
| 41 """ | 42 """ |
| 42 def __init__(self, file_system, object_store): | 43 def __init__(self, file_system, object_store, branch_number=DEFAULT_BRANCH): |
| 43 self._file_system = file_system | 44 self._file_system = file_system |
| 44 self._object_store = object_store | 45 self._object_store = object_store |
| 46 self._branch_number = branch_number | |
| 45 | 47 |
| 46 def Create(self, populate_function, namespace, version=None): | 48 def Create(self, populate_function, namespace, version=None): |
| 47 """Create a CompiledFileSystem that populates the cache by calling | 49 """Create a CompiledFileSystem that populates the cache by calling |
| 48 |populate_function| with (path, data), where |data| is the data that was | 50 |populate_function| with (path, data), where |data| is the data that was |
| 49 fetched from |path|. The keys to the cache are put in the namespace | 51 fetched from |path|. The keys to the cache are put in the namespace |
| 50 specified by |namespace|, and optionally adding |version|. """ | 52 specified by |namespace|, and optionally adding |version|. |
| 53 """ | |
| 51 return CompiledFileSystem(self._file_system, | 54 return CompiledFileSystem(self._file_system, |
| 52 populate_function, | 55 populate_function, |
| 53 self._object_store, | 56 self._object_store, |
| 54 namespace, | 57 namespace, |
| 58 self._branch_number, | |
| 55 version=version) | 59 version=version) |
| 56 | 60 |
| 57 def __init__(self, | 61 def __init__(self, |
| 58 file_system, | 62 file_system, |
| 59 populate_function, | 63 populate_function, |
| 60 object_store, | 64 object_store, |
| 61 namespace, | 65 namespace, |
| 66 branch_number, | |
| 62 version=None): | 67 version=None): |
| 63 self._file_system = file_system | 68 self._file_system = file_system |
| 64 self._populate_function = populate_function | 69 self._populate_function = populate_function |
| 65 self._object_store = object_store | 70 self._object_store = object_store |
| 66 self._namespace = 'CompiledFileSystem.' + namespace | 71 self._namespace = 'CompiledFileSystem.' + namespace |
| 72 self._branch_number = branch_number | |
|
cduvall
2013/03/21 18:43:53
Doesn't the file system already have a branch numb
epeterson
2013/03/25 19:35:11
Done.
| |
| 67 if version is not None: | 73 if version is not None: |
| 68 self._namespace = '%s.%s' % (self._namespace, version) | 74 self._namespace = '%s.%s.%s' % (self._namespace, |
| 75 version, | |
| 76 self._branch_number) | |
| 69 | 77 |
| 70 def _MakeKey(self, key): | 78 def _MakeKey(self, key): |
| 71 return self._namespace + '.' + key | 79 return self._namespace + '.' + key |
| 72 | 80 |
| 73 def _RecursiveList(self, files): | 81 def _RecursiveList(self, files): |
| 74 all_files = files[:] | 82 all_files = files[:] |
| 75 dirs = {} | 83 dirs = {} |
| 76 for filename in files: | 84 for filename in files: |
| 77 if filename.endswith('/'): | 85 if filename.endswith('/'): |
| 78 all_files.remove(filename) | 86 all_files.remove(filename) |
| 79 dirs.update(self._file_system.Read([filename]).Get()) | 87 dirs.update(self._file_system.Read([filename]).Get()) |
| 80 for dir_, files in dirs.iteritems(): | 88 for dir_, files in dirs.iteritems(): |
| 81 all_files.extend(self._RecursiveList([dir_ + f for f in files])) | 89 all_files.extend(self._RecursiveList([dir_ + f for f in files])) |
| 82 return all_files | 90 return all_files |
| 83 | 91 |
| 84 def GetFromFile(self, path, binary=False): | 92 def GetFromFile(self, path, binary=False): |
| 85 """Calls |populate_function| on the contents of the file at |path|. If | 93 """Calls |populate_function| on the contents of the file at |path|. If |
| 86 |binary| is True then the file will be read as binary - but this will only | 94 |binary| is True then the file will be read as binary - but this will only |
| 87 apply for the first time the file is fetched; if already cached, |binary| | 95 apply for the first time the file is fetched; if already cached, |binary| |
| 88 will be ignored. | 96 will be ignored. |
| 89 """ | 97 """ |
| 90 version = self._file_system.Stat(path).version | 98 version = self._file_system.Stat(path).version |
| 91 cache_entry = self._object_store.Get(self._MakeKey(path), | 99 cache_entry = self._object_store.Get(self._MakeKey(path), |
| 92 object_store.FILE_SYSTEM_CACHE, | 100 object_store.FILE_SYSTEM_CACHE, |
| 93 time=0).Get() | 101 time=0).Get() |
| 94 if (cache_entry is not None) and (version == cache_entry.version): | 102 if (cache_entry is not None) and (version == cache_entry.version): |
| 95 return cache_entry._cache_data | 103 return cache_entry._cache_data |
| 96 cache_data = self._populate_function( | 104 try: |
| 97 path, | 105 cache_data = self._populate_function( |
| 98 self._file_system.ReadSingle(path, binary=binary)) | 106 path, |
| 99 self._object_store.Set(self._MakeKey(path), | 107 self._file_system.ReadSingle(path, binary=binary)) |
| 100 _CacheEntry(cache_data, version), | 108 self._object_store.Set(self._MakeKey(path), |
| 101 object_store.FILE_SYSTEM_CACHE, | 109 _CacheEntry(cache_data, version), |
| 102 time=0) | 110 object_store.FILE_SYSTEM_CACHE, |
| 103 return cache_data | 111 time=0) |
| 112 return cache_data | |
| 113 except ValueError as e: | |
| 114 return {} | |
| 115 logging.error("%s" % e) | |
|
cduvall
2013/03/21 18:43:53
Give a better message than this. And single quotes
epeterson
2013/03/25 19:35:11
Done? Does this work for you?
| |
| 104 | 116 |
| 105 def GetFromFileListing(self, path): | 117 def GetFromFileListing(self, path): |
| 106 """Calls |populate_function| on the listing of the files at |path|. | 118 """Calls |populate_function| on the listing of the files at |path|. |
| 107 Assumes that the path given is to a directory. | 119 Assumes that the path given is to a directory. |
| 108 """ | 120 """ |
| 109 if not path.endswith('/'): | 121 if not path.endswith('/'): |
| 110 path += '/' | 122 path += '/' |
| 111 version = self._file_system.Stat(path).version | 123 version = self._file_system.Stat(path).version |
| 112 cache_entry = self._object_store.Get( | 124 cache_entry = self._object_store.Get( |
| 113 self._MakeKey(path), | 125 self._MakeKey(path), |
| 114 object_store.FILE_SYSTEM_CACHE_LISTING, | 126 object_store.FILE_SYSTEM_CACHE_LISTING, |
| 115 time=0).Get() | 127 time=0).Get() |
| 116 if (cache_entry is not None) and (version == cache_entry.version): | 128 if (cache_entry is not None) and (version == cache_entry.version): |
| 117 return cache_entry._cache_data | 129 return cache_entry._cache_data |
| 118 cache_data = self._populate_function( | 130 cache_data = self._populate_function( |
| 119 path, | 131 path, |
| 120 self._RecursiveList( | 132 self._RecursiveList( |
| 121 [path + f for f in self._file_system.ReadSingle(path)])) | 133 [path + f for f in self._file_system.ReadSingle(path)])) |
| 122 self._object_store.Set(self._MakeKey(path), | 134 self._object_store.Set(self._MakeKey(path), |
| 123 _CacheEntry(cache_data, version), | 135 _CacheEntry(cache_data, version), |
| 124 object_store.FILE_SYSTEM_CACHE_LISTING, | 136 object_store.FILE_SYSTEM_CACHE_LISTING, |
| 125 time=0) | 137 time=0) |
| 126 return cache_data | 138 return cache_data |
| OLD | NEW |