OLD | NEW |
---|---|
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import os | 5 import os |
6 | 6 |
7 import object_store | 7 import object_store |
8 import logging | |
8 | 9 |
9 APPS = 'Apps' | 10 APPS = 'Apps' |
10 APPS_FS = 'AppsFileSystem' | 11 APPS_FS = 'AppsFileSystem' |
11 CRON = 'Cron' | 12 CRON = 'Cron' |
12 EXTENSIONS = 'Extensions' | 13 EXTENSIONS = 'Extensions' |
13 EXTENSIONS_FS = 'ExtensionsFileSystem' | 14 EXTENSIONS_FS = 'ExtensionsFileSystem' |
14 CRON_FILE_LISTING = 'Cron.FileListing' | 15 CRON_FILE_LISTING = 'Cron.FileListing' |
15 CRON_GITHUB_INVALIDATION = 'Cron.GithubInvalidation' | 16 CRON_GITHUB_INVALIDATION = 'Cron.GithubInvalidation' |
16 CRON_INVALIDATION = 'Cron.Invalidation' | 17 CRON_INVALIDATION = 'Cron.Invalidation' |
17 HANDLEBAR = 'Handlebar' | 18 HANDLEBAR = 'Handlebar' |
18 IDL = 'IDL' | 19 IDL = 'IDL' |
19 IDL_NO_REFS = 'IDLNoRefs' | 20 IDL_NO_REFS = 'IDLNoRefs' |
20 IDL_NAMES = 'IDLNames' | 21 IDL_NAMES = 'IDLNames' |
21 INTRO = 'Intro' | 22 INTRO = 'Intro' |
22 JSON = 'JSON' | 23 JSON = 'JSON' |
23 JSON_NO_REFS = 'JSONNoRefs' | 24 JSON_NO_REFS = 'JSONNoRefs' |
24 LIST = 'List' | 25 LIST = 'List' |
25 NAMES = 'Names' | 26 NAMES = 'Names' |
26 PERMS = 'Perms' | 27 PERMS = 'Perms' |
27 SIDENAV = 'Sidenav' | 28 SIDENAV = 'Sidenav' |
28 STATIC = 'Static' | 29 STATIC = 'Static' |
29 ZIP = 'Zip' | 30 ZIP = 'Zip' |
30 | 31 DEFAULT_BRANCH = '000' |
cduvall
2013/03/21 18:43:53
Separate this from the namespace constants.
epeterson
2013/03/25 19:35:11
Done.
| |
31 class _CacheEntry(object): | 32 class _CacheEntry(object): |
32 def __init__(self, cache_data, version): | 33 def __init__(self, cache_data, version): |
33 self._cache_data = cache_data | 34 self._cache_data = cache_data |
34 self.version = version | 35 self.version = version |
35 | 36 |
36 class CompiledFileSystem(object): | 37 class CompiledFileSystem(object): |
37 """This class caches FileSystem data that has been processed. | 38 """This class caches FileSystem data that has been processed. |
38 """ | 39 """ |
39 class Factory(object): | 40 class Factory(object): |
40 """A class to build a CompiledFileSystem. | 41 """A class to build a CompiledFileSystem. |
41 """ | 42 """ |
42 def __init__(self, file_system, object_store): | 43 def __init__(self, file_system, object_store, branch_number=DEFAULT_BRANCH): |
43 self._file_system = file_system | 44 self._file_system = file_system |
44 self._object_store = object_store | 45 self._object_store = object_store |
46 self._branch_number = branch_number | |
45 | 47 |
46 def Create(self, populate_function, namespace, version=None): | 48 def Create(self, populate_function, namespace, version=None): |
47 """Create a CompiledFileSystem that populates the cache by calling | 49 """Create a CompiledFileSystem that populates the cache by calling |
48 |populate_function| with (path, data), where |data| is the data that was | 50 |populate_function| with (path, data), where |data| is the data that was |
49 fetched from |path|. The keys to the cache are put in the namespace | 51 fetched from |path|. The keys to the cache are put in the namespace |
50 specified by |namespace|, and optionally adding |version|. """ | 52 specified by |namespace|, and optionally adding |version|. |
53 """ | |
51 return CompiledFileSystem(self._file_system, | 54 return CompiledFileSystem(self._file_system, |
52 populate_function, | 55 populate_function, |
53 self._object_store, | 56 self._object_store, |
54 namespace, | 57 namespace, |
58 self._branch_number, | |
55 version=version) | 59 version=version) |
56 | 60 |
57 def __init__(self, | 61 def __init__(self, |
58 file_system, | 62 file_system, |
59 populate_function, | 63 populate_function, |
60 object_store, | 64 object_store, |
61 namespace, | 65 namespace, |
66 branch_number, | |
62 version=None): | 67 version=None): |
63 self._file_system = file_system | 68 self._file_system = file_system |
64 self._populate_function = populate_function | 69 self._populate_function = populate_function |
65 self._object_store = object_store | 70 self._object_store = object_store |
66 self._namespace = 'CompiledFileSystem.' + namespace | 71 self._namespace = 'CompiledFileSystem.' + namespace |
72 self._branch_number = branch_number | |
cduvall
2013/03/21 18:43:53
Doesn't the file system already have a branch numb
epeterson
2013/03/25 19:35:11
Done.
| |
67 if version is not None: | 73 if version is not None: |
68 self._namespace = '%s.%s' % (self._namespace, version) | 74 self._namespace = '%s.%s.%s' % (self._namespace, |
75 version, | |
76 self._branch_number) | |
69 | 77 |
70 def _MakeKey(self, key): | 78 def _MakeKey(self, key): |
71 return self._namespace + '.' + key | 79 return self._namespace + '.' + key |
72 | 80 |
73 def _RecursiveList(self, files): | 81 def _RecursiveList(self, files): |
74 all_files = files[:] | 82 all_files = files[:] |
75 dirs = {} | 83 dirs = {} |
76 for filename in files: | 84 for filename in files: |
77 if filename.endswith('/'): | 85 if filename.endswith('/'): |
78 all_files.remove(filename) | 86 all_files.remove(filename) |
79 dirs.update(self._file_system.Read([filename]).Get()) | 87 dirs.update(self._file_system.Read([filename]).Get()) |
80 for dir_, files in dirs.iteritems(): | 88 for dir_, files in dirs.iteritems(): |
81 all_files.extend(self._RecursiveList([dir_ + f for f in files])) | 89 all_files.extend(self._RecursiveList([dir_ + f for f in files])) |
82 return all_files | 90 return all_files |
83 | 91 |
84 def GetFromFile(self, path, binary=False): | 92 def GetFromFile(self, path, binary=False): |
85 """Calls |populate_function| on the contents of the file at |path|. If | 93 """Calls |populate_function| on the contents of the file at |path|. If |
86 |binary| is True then the file will be read as binary - but this will only | 94 |binary| is True then the file will be read as binary - but this will only |
87 apply for the first time the file is fetched; if already cached, |binary| | 95 apply for the first time the file is fetched; if already cached, |binary| |
88 will be ignored. | 96 will be ignored. |
89 """ | 97 """ |
90 version = self._file_system.Stat(path).version | 98 version = self._file_system.Stat(path).version |
91 cache_entry = self._object_store.Get(self._MakeKey(path), | 99 cache_entry = self._object_store.Get(self._MakeKey(path), |
92 object_store.FILE_SYSTEM_CACHE, | 100 object_store.FILE_SYSTEM_CACHE, |
93 time=0).Get() | 101 time=0).Get() |
94 if (cache_entry is not None) and (version == cache_entry.version): | 102 if (cache_entry is not None) and (version == cache_entry.version): |
95 return cache_entry._cache_data | 103 return cache_entry._cache_data |
96 cache_data = self._populate_function( | 104 try: |
97 path, | 105 cache_data = self._populate_function( |
98 self._file_system.ReadSingle(path, binary=binary)) | 106 path, |
99 self._object_store.Set(self._MakeKey(path), | 107 self._file_system.ReadSingle(path, binary=binary)) |
100 _CacheEntry(cache_data, version), | 108 self._object_store.Set(self._MakeKey(path), |
101 object_store.FILE_SYSTEM_CACHE, | 109 _CacheEntry(cache_data, version), |
102 time=0) | 110 object_store.FILE_SYSTEM_CACHE, |
103 return cache_data | 111 time=0) |
112 return cache_data | |
113 except ValueError as e: | |
114 return {} | |
115 logging.error("%s" % e) | |
cduvall
2013/03/21 18:43:53
Give a better message than this. And single quotes
epeterson
2013/03/25 19:35:11
Done? Does this work for you?
| |
104 | 116 |
105 def GetFromFileListing(self, path): | 117 def GetFromFileListing(self, path): |
106 """Calls |populate_function| on the listing of the files at |path|. | 118 """Calls |populate_function| on the listing of the files at |path|. |
107 Assumes that the path given is to a directory. | 119 Assumes that the path given is to a directory. |
108 """ | 120 """ |
109 if not path.endswith('/'): | 121 if not path.endswith('/'): |
110 path += '/' | 122 path += '/' |
111 version = self._file_system.Stat(path).version | 123 version = self._file_system.Stat(path).version |
112 cache_entry = self._object_store.Get( | 124 cache_entry = self._object_store.Get( |
113 self._MakeKey(path), | 125 self._MakeKey(path), |
114 object_store.FILE_SYSTEM_CACHE_LISTING, | 126 object_store.FILE_SYSTEM_CACHE_LISTING, |
115 time=0).Get() | 127 time=0).Get() |
116 if (cache_entry is not None) and (version == cache_entry.version): | 128 if (cache_entry is not None) and (version == cache_entry.version): |
117 return cache_entry._cache_data | 129 return cache_entry._cache_data |
118 cache_data = self._populate_function( | 130 cache_data = self._populate_function( |
119 path, | 131 path, |
120 self._RecursiveList( | 132 self._RecursiveList( |
121 [path + f for f in self._file_system.ReadSingle(path)])) | 133 [path + f for f in self._file_system.ReadSingle(path)])) |
122 self._object_store.Set(self._MakeKey(path), | 134 self._object_store.Set(self._MakeKey(path), |
123 _CacheEntry(cache_data, version), | 135 _CacheEntry(cache_data, version), |
124 object_store.FILE_SYSTEM_CACHE_LISTING, | 136 object_store.FILE_SYSTEM_CACHE_LISTING, |
125 time=0) | 137 time=0) |
126 return cache_data | 138 return cache_data |
OLD | NEW |