OLD | NEW |
| (Empty) |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 """Defines the PersistentMixIn utility class to easily convert classes to and | |
6 from dict for serialization. | |
7 | |
8 This class is aimed at json-compatible serialization, so it supports the limited | |
9 set of structures supported by json; strings, numbers as int or float, list and | |
10 dictionaries. | |
11 | |
12 PersistentMixIn._persistent_members() returns a dict of each member with the | |
13 tuple of expected types. Each member can be decoded in multiple types, for | |
14 example, a subversion revision number could have (None, int, str), meaning that | |
15 the revision could be None, when not known, an int or the int as a string | |
16 representation. The tuple is listed in the prefered order of conversions. | |
17 | |
18 Composites types that cannot be represented exactly in json like tuple, set and | |
19 frozenset are converted from and back to list automatically. Any class instance | |
20 that has been serialized can be unserialized in the same class instance or into | |
21 a bare dict. | |
22 | |
23 See tests/model_tests.py for examples. | |
24 """ | |
25 | |
26 import json | |
27 import logging | |
28 import os | |
29 | |
30 # Set in the output dict to be able to know which class was serialized to help | |
31 # deserialization. | |
32 TYPE_FLAG = '__persistent_type__' | |
33 | |
34 # Marker to tell the deserializer that we don't know the expected type, used in | |
35 # composite types. | |
36 _UNKNOWN = object() | |
37 | |
38 | |
39 def as_dict(value): | |
40 """Recursively converts an object into a dictionary. | |
41 | |
42 Converts tuple,set,frozenset into list and recursively process each items. | |
43 """ | |
44 if hasattr(value, 'as_dict') and callable(value.as_dict): | |
45 return value.as_dict() | |
46 elif isinstance(value, (list, tuple, set, frozenset)): | |
47 return [as_dict(v) for v in value] | |
48 elif isinstance(value, dict): | |
49 return dict((as_dict(k), as_dict(v)) | |
50 for k, v in value.iteritems()) | |
51 elif isinstance(value, (bool, float, int, basestring)) or value is None: | |
52 return value | |
53 else: | |
54 raise AttributeError('Can\'t type %s into a dictionary' % type(value)) | |
55 | |
56 | |
57 def _inner_from_dict(name, value, member_types): | |
58 """Recursively regenerates an object. | |
59 | |
60 For each of the allowable types, try to convert it. If None is an allowable | |
61 type, any data that can't be parsed will be parsed as None and will be | |
62 silently discarded. Otherwise, an exception will be raise. | |
63 """ | |
64 logging.debug('_inner_from_dict(%s, %r, %s)', name, value, member_types) | |
65 result = None | |
66 if member_types is _UNKNOWN: | |
67 # Use guesswork a bit more and accept anything. | |
68 if isinstance(value, dict): | |
69 if TYPE_FLAG in value: | |
70 result = PersistentMixIn.from_dict(value, _UNKNOWN) | |
71 else: | |
72 # Unserialize it as a raw dict. | |
73 result = dict( | |
74 (_inner_from_dict(None, k, _UNKNOWN), | |
75 _inner_from_dict(None, v, _UNKNOWN)) | |
76 for k, v in value.iteritems()) | |
77 elif isinstance(value, list): | |
78 # All of these are serialized to list. | |
79 result = [_inner_from_dict(None, v, _UNKNOWN) for v in value] | |
80 elif isinstance(value, (bool, float, int, unicode)): | |
81 result = value | |
82 else: | |
83 raise TypeError('No idea how to convert %r' % value) | |
84 else: | |
85 for member_type in member_types: | |
86 # Explicitly leave None out of this loop. | |
87 if issubclass(member_type, PersistentMixIn): | |
88 if isinstance(value, dict) and TYPE_FLAG in value: | |
89 result = PersistentMixIn.from_dict(value, member_type) | |
90 break | |
91 elif member_type is dict: | |
92 if isinstance(value, dict): | |
93 result = dict( | |
94 (_inner_from_dict(None, k, _UNKNOWN), | |
95 _inner_from_dict(None, v, _UNKNOWN)) | |
96 for k, v in value.iteritems()) | |
97 break | |
98 elif member_type in (list, tuple, set, frozenset): | |
99 # All of these are serialized to list. | |
100 if isinstance(value, list): | |
101 result = member_type( | |
102 _inner_from_dict(None, v, _UNKNOWN) for v in value) | |
103 break | |
104 elif member_type in (bool, float, int, str, unicode): | |
105 if isinstance(value, member_type): | |
106 result = member_type(value) | |
107 break | |
108 elif member_type is None.__class__ and value is None: | |
109 result = None | |
110 break | |
111 else: | |
112 logging.info( | |
113 'Ignored %s: didn\'t fit types %s; %s', | |
114 name, | |
115 ', '.join(i.__name__ for i in member_types), | |
116 repr(value)[:200]) | |
117 _check_type_value(name, result, member_types) | |
118 return result | |
119 | |
120 | |
121 def to_yaml(obj): | |
122 """Converts a PersistentMixIn into a yaml-inspired format. | |
123 | |
124 Warning: Not unit tested, use at your own risk! | |
125 """ | |
126 def align(x): | |
127 y = x.splitlines(True) | |
128 if len(y) > 1: | |
129 return ''.join(y[0:1] + [' ' + z for z in y[1:]]) | |
130 return x | |
131 def align_value(x): | |
132 if '\n' in x: | |
133 return '\n ' + align(x) | |
134 return x | |
135 | |
136 if hasattr(obj, 'as_dict') and callable(obj.as_dict): | |
137 out = (to_yaml(obj.as_dict()),) | |
138 elif isinstance(obj, (bool, float, int, unicode)) or obj is None: | |
139 out = (align(str(obj)),) | |
140 elif isinstance(obj, dict): | |
141 if TYPE_FLAG in obj: | |
142 out = ['%s:' % obj[TYPE_FLAG]] | |
143 else: | |
144 out = [] | |
145 for k, v in obj.iteritems(): | |
146 # Skips many members resolving to bool() == False | |
147 if k.startswith('__') or v in (None, '', False, 0): | |
148 continue | |
149 r = align_value(to_yaml(v)) | |
150 if not r: | |
151 continue | |
152 out.append('- %s: %s' % (k, r)) | |
153 elif hasattr(obj, '__iter__') and callable(obj.__iter__): | |
154 out = ['- %s' % align(to_yaml(x)) for x in obj] | |
155 else: | |
156 out = ('%s' % obj.__class__.__name__,) | |
157 return '\n'.join(out) | |
158 | |
159 | |
160 def _default_value(member_types): | |
161 """Returns an instance of the first allowed type. Special case None.""" | |
162 if member_types[0] is None.__class__: | |
163 return None | |
164 else: | |
165 return member_types[0]() | |
166 | |
167 | |
168 def _check_type_value(name, value, member_types): | |
169 """Raises a TypeError exception if value is not one of the allowed types in | |
170 member_types. | |
171 """ | |
172 if not isinstance(value, member_types): | |
173 prefix = '%s e' % name if name else 'E' | |
174 raise TypeError( | |
175 '%sxpected type(s) %s; got %r' % | |
176 (prefix, ', '.join(i.__name__ for i in member_types), value)) | |
177 | |
178 | |
179 | |
180 class PersistentMixIn(object): | |
181 """Class to be used as a base class to persistent data in a simplistic way. | |
182 | |
183 Persistent class member needs to be set to a tuple containing the instance | |
184 member variable that needs to be saved or loaded. The first item will be | |
185 default value, e.g.: | |
186 foo = (None, str, dict) | |
187 Will default initialize self.foo to None. | |
188 """ | |
189 # Cache of all the subclasses of PersistentMixIn. | |
190 __persistent_classes_cache = None | |
191 | |
192 _read_only = False | |
193 | |
194 def __init__(self, **kwargs): | |
195 """Initializes with the default members.""" | |
196 super(PersistentMixIn, self).__init__() | |
197 persistent_members = self._persistent_members() | |
198 for member, member_types in persistent_members.iteritems(): | |
199 if member in kwargs: | |
200 value = kwargs.pop(member) | |
201 if isinstance(value, str): | |
202 # Assume UTF-8 all the time. Note: This is explicitly when the object | |
203 # is constructed in the code. This code path is never used when | |
204 # deserializing the object. | |
205 value = value.decode('utf-8') | |
206 else: | |
207 value = _default_value(member_types) | |
208 _check_type_value(member, value, member_types) | |
209 setattr(self, member, value) | |
210 if kwargs: | |
211 raise AttributeError('Received unexpected initializers: %s' % kwargs) | |
212 | |
213 def __setattr__(self, name, value): | |
214 """Enforces immutability if cls._read_only is True.""" | |
215 if self._read_only: | |
216 raise TypeError() | |
217 return super(PersistentMixIn, self).__setattr__(name, value) | |
218 | |
219 @classmethod | |
220 def _persistent_members(cls): | |
221 """Returns the persistent items as a dict. | |
222 | |
223 Each entry value can be a tuple when the member can be assigned different | |
224 types. | |
225 """ | |
226 # Note that here, cls is the subclass, not PersistentMixIn. | |
227 # TODO(maruel): Cache the results. It's tricky because setting | |
228 # cls.__persistent_members_cache on a class will implicitly set it on its | |
229 # subclass. So in a class hierarchy with A -> B -> PersistentMixIn, calling | |
230 # B()._persistent_members() will incorrectly set the cache for A. | |
231 persistent_members_cache = {} | |
232 # Enumerate on the subclass, not on an instance. | |
233 for item in dir(cls): | |
234 if item.startswith('_'): | |
235 continue | |
236 item_value = getattr(cls, item) | |
237 if isinstance(item_value, type): | |
238 item_value = (item_value,) | |
239 if not isinstance(item_value, tuple): | |
240 continue | |
241 if not all(i is None or i.__class__ == type for i in item_value): | |
242 continue | |
243 if any(i is str for i in item_value): | |
244 raise TypeError( | |
245 '%s is type \'str\' which is currently not supported' % item) | |
246 item_value = tuple( | |
247 f if f is not None else None.__class__ for f in item_value) | |
248 persistent_members_cache[item] = item_value | |
249 return persistent_members_cache | |
250 | |
251 @staticmethod | |
252 def _get_subclass(typename): | |
253 """Returns the PersistentMixIn subclass with the name |typename|.""" | |
254 subclass = None | |
255 if PersistentMixIn.__persistent_classes_cache is not None: | |
256 subclass = PersistentMixIn.__persistent_classes_cache.get(typename) | |
257 if not subclass: | |
258 # Get the subclasses recursively. | |
259 PersistentMixIn.__persistent_classes_cache = {} | |
260 def recurse(c): | |
261 for s in c.__subclasses__(): | |
262 assert s.__name__ not in PersistentMixIn.__persistent_classes_cache | |
263 PersistentMixIn.__persistent_classes_cache[s.__name__] = s | |
264 recurse(s) | |
265 recurse(PersistentMixIn) | |
266 | |
267 subclass = PersistentMixIn.__persistent_classes_cache.get(typename) | |
268 if not subclass: | |
269 raise KeyError('Couldn\'t find type %s' % typename) | |
270 return subclass | |
271 | |
272 def as_dict(self): | |
273 """Create a dictionary out of this object, i.e. Serialize the object.""" | |
274 out = {} | |
275 for member, member_types in self._persistent_members().iteritems(): | |
276 value = getattr(self, member) | |
277 _check_type_value(member, value, member_types) | |
278 out[member] = as_dict(value) | |
279 out[TYPE_FLAG] = self.__class__.__name__ | |
280 return out | |
281 | |
282 @staticmethod | |
283 def from_dict(data, subclass=_UNKNOWN): | |
284 """Returns an instance of a class inheriting from PersistentMixIn, | |
285 initialized with 'data' dict, i.e. Deserialize the object. | |
286 """ | |
287 logging.debug('from_dict(%r, %s)', data, subclass) | |
288 if subclass is _UNKNOWN: | |
289 subclass = PersistentMixIn._get_subclass(data[TYPE_FLAG]) | |
290 # This initializes the instance with the default values. | |
291 | |
292 # pylint: disable=W0212 | |
293 kwargs = {} | |
294 for member, member_types in subclass._persistent_members().iteritems(): | |
295 if member in data: | |
296 try: | |
297 value = _inner_from_dict(member, data[member], member_types) | |
298 except TypeError: | |
299 # pylint: disable=E1103 | |
300 logging.error( | |
301 'Failed to instantiate %s because of member %s', | |
302 subclass.__name__, member) | |
303 raise | |
304 else: | |
305 value = _default_value(member_types) | |
306 _check_type_value(member, value, member_types) | |
307 kwargs[member] = value | |
308 try: | |
309 obj = subclass(**kwargs) | |
310 except TypeError: | |
311 # pylint: disable=E1103 | |
312 logging.error('Failed to instantiate %s: %r', subclass.__name__, kwargs) | |
313 raise | |
314 assert isinstance(obj, PersistentMixIn) and obj.__class__ != PersistentMixIn | |
315 return obj | |
316 | |
317 def __str__(self): | |
318 return to_yaml(self) | |
319 | |
320 def __eq__(self, _): | |
321 raise TypeError() | |
322 | |
323 # pylint: disable=R0201 | |
324 def __ne__(self, _): | |
325 raise TypeError() | |
326 | |
327 | |
328 def is_equivalent(lhs, rhs): | |
329 """Implements the equivalent of __eq__. | |
330 | |
331 The reason for not implementing __eq__ is to not encourage bad behavior by | |
332 implicitly and recursively using __eq__() in a list().remove() call. | |
333 """ | |
334 # pylint: disable=W0212 | |
335 if lhs._persistent_members() != rhs._persistent_members(): | |
336 return False | |
337 for i in lhs._persistent_members(): | |
338 if getattr(lhs, i) != getattr(rhs, i): | |
339 return False | |
340 return True | |
341 | |
342 | |
343 def immutable(func): | |
344 """Member function decorators that convert 'self' to an immutable object. | |
345 | |
346 Member functions of the object can't be called unless they are immutable too. | |
347 Properties can be looked up, this function assumes properties do not mutate | |
348 the object. | |
349 | |
350 Note: a user can still call classmethod and do mutation on the class, or they | |
351 can lookup a member object and mutate this one. Don't be silly. | |
352 """ | |
353 class Immutable(object): | |
354 def __init__(self, obj): | |
355 object.__setattr__(self, '__ref', obj) | |
356 | |
357 def __getattribute__(self, name): | |
358 ref = object.__getattribute__(self, '__ref') | |
359 value = getattr(ref, name) | |
360 if not callable(value): | |
361 return value | |
362 if getattr(value, 'is_immutable', None): | |
363 # It is immutable too. | |
364 return value | |
365 if getattr(value, 'im_self', None) == None: | |
366 # It is static. | |
367 return value | |
368 raise TypeError( | |
369 'Can\'t call mutable member function \'%s\' on an immutable ' | |
370 'instance of %s' % (name, ref.__class__.__name__)) | |
371 | |
372 def __setattr__(self, name, _value): | |
373 ref = object.__getattribute__(self, '__ref') | |
374 raise TypeError( | |
375 'Can\'t change attribute \'%s\' on an immutable instance of \'%s\'' % | |
376 (name, ref.__class__.__name__)) | |
377 | |
378 def __delattr__(self, name): | |
379 ref = object.__getattribute__(self, '__ref') | |
380 raise TypeError( | |
381 'Can\'t delete attribute \'%s\' on an immutable instance of \'%s\'' % | |
382 (name, ref.__class__.__name__)) | |
383 | |
384 def hook(self, *args, **kwargs): | |
385 return func(Immutable(self), *args, **kwargs) | |
386 | |
387 hook.is_immutable = True | |
388 return hook | |
389 | |
390 | |
391 def load_from_json_file(filename): | |
392 """Loads one object from a JSON file.""" | |
393 with open(filename, 'r') as f: | |
394 return PersistentMixIn.from_dict(json.load(f)) | |
395 | |
396 | |
397 def save_to_json_file(filename, obj): | |
398 """Save one object in a JSON file.""" | |
399 try: | |
400 old = filename + '.old' | |
401 if os.path.exists(filename): | |
402 os.rename(filename, old) | |
403 finally: | |
404 with open(filename, 'wb') as f: | |
405 json.dump(obj.as_dict(), f, sort_keys=True, indent=2) | |
406 f.write('\n') | |
OLD | NEW |