| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- test-case-name: twisted.test.test_persisted -*- | |
| 2 | |
| 3 # Copyright (c) 2001-2004 Twisted Matrix Laboratories. | |
| 4 # See LICENSE for details. | |
| 5 | |
| 6 | |
| 7 | |
| 8 """ | |
| 9 AOT: Abstract Object Trees | |
| 10 The source-code-marshallin'est abstract-object-serializin'est persister | |
| 11 this side of Marmalade! | |
| 12 """ | |
| 13 | |
| 14 import types, new, string, copy_reg, tokenize, re | |
| 15 | |
| 16 from twisted.python import reflect, log | |
| 17 from twisted.persisted import crefutil | |
| 18 | |
| 19 ########################### | |
| 20 # Abstract Object Classes # | |
| 21 ########################### | |
| 22 | |
| 23 #"\0" in a getSource means "insert variable-width indention here". | |
| 24 #see `indentify'. | |
| 25 | |
| 26 class Named: | |
| 27 def __init__(self, name): | |
| 28 self.name = name | |
| 29 | |
| 30 class Class(Named): | |
| 31 def getSource(self): | |
| 32 return "Class(%r)" % self.name | |
| 33 | |
| 34 class Function(Named): | |
| 35 def getSource(self): | |
| 36 return "Function(%r)" % self.name | |
| 37 | |
| 38 class Module(Named): | |
| 39 def getSource(self): | |
| 40 return "Module(%r)" % self.name | |
| 41 | |
| 42 | |
| 43 class InstanceMethod: | |
| 44 def __init__(self, name, klass, inst): | |
| 45 if not (isinstance(inst, Ref) or isinstance(inst, Instance) or isinstanc
e(inst, Deref)): | |
| 46 raise TypeError("%s isn't an Instance, Ref, or Deref!" % inst) | |
| 47 self.name = name | |
| 48 self.klass = klass | |
| 49 self.instance = inst | |
| 50 | |
| 51 def getSource(self): | |
| 52 return "InstanceMethod(%r, %r, \n\0%s)" % (self.name, self.klass, pretti
fy(self.instance)) | |
| 53 | |
| 54 | |
| 55 class _NoStateObj: | |
| 56 pass | |
| 57 NoStateObj = _NoStateObj() | |
| 58 | |
| 59 _SIMPLE_BUILTINS = [ | |
| 60 types.StringType, types.UnicodeType, types.IntType, types.FloatType, | |
| 61 types.ComplexType, types.LongType, types.NoneType, types.SliceType, | |
| 62 types.EllipsisType] | |
| 63 | |
| 64 try: | |
| 65 _SIMPLE_BUILTINS.append(types.BooleanType) | |
| 66 except AttributeError: | |
| 67 pass | |
| 68 | |
| 69 class Instance: | |
| 70 def __init__(self, className, __stateObj__=NoStateObj, **state): | |
| 71 if not isinstance(className, types.StringType): | |
| 72 raise TypeError("%s isn't a string!" % className) | |
| 73 self.klass = className | |
| 74 if __stateObj__ is not NoStateObj: | |
| 75 self.state = __stateObj__ | |
| 76 self.stateIsDict = 0 | |
| 77 else: | |
| 78 self.state = state | |
| 79 self.stateIsDict = 1 | |
| 80 | |
| 81 def getSource(self): | |
| 82 #XXX make state be foo=bar instead of a dict. | |
| 83 if self.stateIsDict: | |
| 84 stateDict = self.state | |
| 85 elif isinstance(self.state, Ref) and isinstance(self.state.obj, types.Di
ctType): | |
| 86 stateDict = self.state.obj | |
| 87 else: | |
| 88 stateDict = None | |
| 89 if stateDict is not None: | |
| 90 try: | |
| 91 return "Instance(%r, %s)" % (self.klass, dictToKW(stateDict)) | |
| 92 except NonFormattableDict: | |
| 93 return "Instance(%r, %s)" % (self.klass, prettify(stateDict)) | |
| 94 return "Instance(%r, %s)" % (self.klass, prettify(self.state)) | |
| 95 | |
| 96 class Ref: | |
| 97 | |
| 98 def __init__(self, *args): | |
| 99 #blargh, lame. | |
| 100 if len(args) == 2: | |
| 101 self.refnum = args[0] | |
| 102 self.obj = args[1] | |
| 103 elif not args: | |
| 104 self.refnum = None | |
| 105 self.obj = None | |
| 106 | |
| 107 def setRef(self, num): | |
| 108 if self.refnum: | |
| 109 raise ValueError("Error setting id %s, I already have %s" % (num, se
lf.refnum)) | |
| 110 self.refnum = num | |
| 111 | |
| 112 def setObj(self, obj): | |
| 113 if self.obj: | |
| 114 raise ValueError("Error setting obj %s, I already have %s" % (obj, s
elf.obj)) | |
| 115 self.obj = obj | |
| 116 | |
| 117 def getSource(self): | |
| 118 if self.obj is None: | |
| 119 raise RuntimeError("Don't try to display me before setting an object
on me!") | |
| 120 if self.refnum: | |
| 121 return "Ref(%d, \n\0%s)" % (self.refnum, prettify(self.obj)) | |
| 122 return prettify(self.obj) | |
| 123 | |
| 124 | |
| 125 class Deref: | |
| 126 def __init__(self, num): | |
| 127 self.refnum = num | |
| 128 | |
| 129 def getSource(self): | |
| 130 return "Deref(%d)" % self.refnum | |
| 131 | |
| 132 __repr__ = getSource | |
| 133 | |
| 134 | |
| 135 class Copyreg: | |
| 136 def __init__(self, loadfunc, state): | |
| 137 self.loadfunc = loadfunc | |
| 138 self.state = state | |
| 139 | |
| 140 def getSource(self): | |
| 141 return "Copyreg(%r, %s)" % (self.loadfunc, prettify(self.state)) | |
| 142 | |
| 143 | |
| 144 | |
| 145 ############### | |
| 146 # Marshalling # | |
| 147 ############### | |
| 148 | |
| 149 | |
| 150 def getSource(ao): | |
| 151 """Pass me an AO, I'll return a nicely-formatted source representation.""" | |
| 152 return indentify("app = " + prettify(ao)) | |
| 153 | |
| 154 | |
| 155 class NonFormattableDict(Exception): | |
| 156 """A dictionary was not formattable. | |
| 157 """ | |
| 158 | |
| 159 r = re.compile('[a-zA-Z_][a-zA-Z0-9_]*$') | |
| 160 | |
| 161 def dictToKW(d): | |
| 162 out = [] | |
| 163 items = d.items() | |
| 164 items.sort() | |
| 165 for k,v in items: | |
| 166 if not isinstance(k, types.StringType): | |
| 167 raise NonFormattableDict("%r ain't a string" % k) | |
| 168 if not r.match(k): | |
| 169 raise NonFormattableDict("%r ain't an identifier" % k) | |
| 170 out.append( | |
| 171 "\n\0%s=%s," % (k, prettify(v)) | |
| 172 ) | |
| 173 return string.join(out, '') | |
| 174 | |
| 175 | |
| 176 def prettify(obj): | |
| 177 if hasattr(obj, 'getSource'): | |
| 178 return obj.getSource() | |
| 179 else: | |
| 180 #basic type | |
| 181 t = type(obj) | |
| 182 | |
| 183 if t in _SIMPLE_BUILTINS: | |
| 184 return repr(obj) | |
| 185 | |
| 186 elif t is types.DictType: | |
| 187 out = ['{'] | |
| 188 for k,v in obj.items(): | |
| 189 out.append('\n\0%s: %s,' % (prettify(k), prettify(v))) | |
| 190 out.append(len(obj) and '\n\0}' or '}') | |
| 191 return string.join(out, '') | |
| 192 | |
| 193 elif t is types.ListType: | |
| 194 out = ["["] | |
| 195 for x in obj: | |
| 196 out.append('\n\0%s,' % prettify(x)) | |
| 197 out.append(len(obj) and '\n\0]' or ']') | |
| 198 return string.join(out, '') | |
| 199 | |
| 200 elif t is types.TupleType: | |
| 201 out = ["("] | |
| 202 for x in obj: | |
| 203 out.append('\n\0%s,' % prettify(x)) | |
| 204 out.append(len(obj) and '\n\0)' or ')') | |
| 205 return string.join(out, '') | |
| 206 else: | |
| 207 raise TypeError("Unsupported type %s when trying to prettify %s." %
(t, obj)) | |
| 208 | |
| 209 def indentify(s): | |
| 210 out = [] | |
| 211 stack = [] | |
| 212 def eater(type, val, r, c, l, out=out, stack=stack): | |
| 213 #import sys | |
| 214 #sys.stdout.write(val) | |
| 215 if val in ['[', '(', '{']: | |
| 216 stack.append(val) | |
| 217 elif val in [']', ')', '}']: | |
| 218 stack.pop() | |
| 219 if val == '\0': | |
| 220 out.append(' '*len(stack)) | |
| 221 else: | |
| 222 out.append(val) | |
| 223 l = ['', s] | |
| 224 tokenize.tokenize(l.pop, eater) | |
| 225 return string.join(out, '') | |
| 226 | |
| 227 | |
| 228 | |
| 229 | |
| 230 | |
| 231 ########### | |
| 232 # Unjelly # | |
| 233 ########### | |
| 234 | |
| 235 def unjellyFromAOT(aot): | |
| 236 """ | |
| 237 Pass me an Abstract Object Tree, and I'll unjelly it for you. | |
| 238 """ | |
| 239 return AOTUnjellier().unjelly(aot) | |
| 240 | |
| 241 def unjellyFromSource(stringOrFile): | |
| 242 """ | |
| 243 Pass me a string of code or a filename that defines an 'app' variable (in | |
| 244 terms of Abstract Objects!), and I'll execute it and unjelly the resulting | |
| 245 AOT for you, returning a newly unpersisted Application object! | |
| 246 """ | |
| 247 | |
| 248 ns = {"Instance": Instance, | |
| 249 "InstanceMethod": InstanceMethod, | |
| 250 "Class": Class, | |
| 251 "Function": Function, | |
| 252 "Module": Module, | |
| 253 "Ref": Ref, | |
| 254 "Deref": Deref, | |
| 255 "Copyreg": Copyreg, | |
| 256 } | |
| 257 | |
| 258 if hasattr(stringOrFile, "read"): | |
| 259 exec stringOrFile.read() in ns | |
| 260 else: | |
| 261 exec stringOrFile in ns | |
| 262 | |
| 263 if ns.has_key('app'): | |
| 264 return unjellyFromAOT(ns['app']) | |
| 265 else: | |
| 266 raise ValueError("%s needs to define an 'app', it didn't!" % stringOrFil
e) | |
| 267 | |
| 268 | |
| 269 class AOTUnjellier: | |
| 270 """I handle the unjellying of an Abstract Object Tree. | |
| 271 See AOTUnjellier.unjellyAO | |
| 272 """ | |
| 273 def __init__(self): | |
| 274 self.references = {} | |
| 275 self.stack = [] | |
| 276 self.afterUnjelly = [] | |
| 277 | |
| 278 ## | |
| 279 # unjelly helpers (copied pretty much directly from marmalade XXX refactor) | |
| 280 ## | |
| 281 def unjellyLater(self, node): | |
| 282 """Unjelly a node, later. | |
| 283 """ | |
| 284 d = crefutil._Defer() | |
| 285 self.unjellyInto(d, 0, node) | |
| 286 return d | |
| 287 | |
| 288 def unjellyInto(self, obj, loc, ao): | |
| 289 """Utility method for unjellying one object into another. | |
| 290 This automates the handling of backreferences. | |
| 291 """ | |
| 292 o = self.unjellyAO(ao) | |
| 293 obj[loc] = o | |
| 294 if isinstance(o, crefutil.NotKnown): | |
| 295 o.addDependant(obj, loc) | |
| 296 return o | |
| 297 | |
| 298 def callAfter(self, callable, result): | |
| 299 if isinstance(result, crefutil.NotKnown): | |
| 300 l = [None] | |
| 301 result.addDependant(l, 1) | |
| 302 else: | |
| 303 l = [result] | |
| 304 self.afterUnjelly.append((callable, l)) | |
| 305 | |
| 306 def unjellyAttribute(self, instance, attrName, ao): | |
| 307 #XXX this is unused???? | |
| 308 """Utility method for unjellying into instances of attributes. | |
| 309 | |
| 310 Use this rather than unjellyAO unless you like surprising bugs! | |
| 311 Alternatively, you can use unjellyInto on your instance's __dict__. | |
| 312 """ | |
| 313 self.unjellyInto(instance.__dict__, attrName, ao) | |
| 314 | |
| 315 def unjellyAO(self, ao): | |
| 316 """Unjelly an Abstract Object and everything it contains. | |
| 317 I return the real object. | |
| 318 """ | |
| 319 self.stack.append(ao) | |
| 320 t = type(ao) | |
| 321 if t is types.InstanceType: | |
| 322 #Abstract Objects | |
| 323 c = ao.__class__ | |
| 324 if c is Module: | |
| 325 return reflect.namedModule(ao.name) | |
| 326 | |
| 327 elif c in [Class, Function] or issubclass(c, type): | |
| 328 return reflect.namedObject(ao.name) | |
| 329 | |
| 330 elif c is InstanceMethod: | |
| 331 im_name = ao.name | |
| 332 im_class = reflect.namedObject(ao.klass) | |
| 333 im_self = self.unjellyAO(ao.instance) | |
| 334 if im_name in im_class.__dict__: | |
| 335 if im_self is None: | |
| 336 return getattr(im_class, im_name) | |
| 337 elif isinstance(im_self, crefutil.NotKnown): | |
| 338 return crefutil._InstanceMethod(im_name, im_self, im_cla
ss) | |
| 339 else: | |
| 340 return new.instancemethod(im_class.__dict__[im_name], | |
| 341 im_self, | |
| 342 im_class) | |
| 343 else: | |
| 344 raise TypeError("instance method changed") | |
| 345 | |
| 346 elif c is Instance: | |
| 347 klass = reflect.namedObject(ao.klass) | |
| 348 state = self.unjellyAO(ao.state) | |
| 349 if hasattr(klass, "__setstate__"): | |
| 350 inst = new.instance(klass, {}) | |
| 351 self.callAfter(inst.__setstate__, state) | |
| 352 else: | |
| 353 inst = new.instance(klass, state) | |
| 354 return inst | |
| 355 | |
| 356 elif c is Ref: | |
| 357 o = self.unjellyAO(ao.obj) #THIS IS CHANGING THE REF OMG | |
| 358 refkey = ao.refnum | |
| 359 ref = self.references.get(refkey) | |
| 360 if ref is None: | |
| 361 self.references[refkey] = o | |
| 362 elif isinstance(ref, crefutil.NotKnown): | |
| 363 ref.resolveDependants(o) | |
| 364 self.references[refkey] = o | |
| 365 elif refkey is None: | |
| 366 # This happens when you're unjellying from an AOT not read f
rom source | |
| 367 pass | |
| 368 else: | |
| 369 raise ValueError("Multiple references with the same ID: %s,
%s, %s!" % (ref, refkey, ao)) | |
| 370 return o | |
| 371 | |
| 372 elif c is Deref: | |
| 373 num = ao.refnum | |
| 374 ref = self.references.get(num) | |
| 375 if ref is None: | |
| 376 der = crefutil._Dereference(num) | |
| 377 self.references[num] = der | |
| 378 return der | |
| 379 return ref | |
| 380 | |
| 381 elif c is Copyreg: | |
| 382 loadfunc = reflect.namedObject(ao.loadfunc) | |
| 383 d = self.unjellyLater(ao.state).addCallback( | |
| 384 lambda result, _l: apply(_l, result), loadfunc) | |
| 385 return d | |
| 386 | |
| 387 #Types | |
| 388 | |
| 389 elif t in _SIMPLE_BUILTINS: | |
| 390 return ao | |
| 391 | |
| 392 elif t is types.ListType: | |
| 393 l = [] | |
| 394 for x in ao: | |
| 395 l.append(None) | |
| 396 self.unjellyInto(l, len(l)-1, x) | |
| 397 return l | |
| 398 | |
| 399 elif t is types.TupleType: | |
| 400 l = [] | |
| 401 tuple_ = tuple | |
| 402 for x in ao: | |
| 403 l.append(None) | |
| 404 if isinstance(self.unjellyInto(l, len(l)-1, x), crefutil.NotKnow
n): | |
| 405 tuple_ = crefutil._Tuple | |
| 406 return tuple_(l) | |
| 407 | |
| 408 elif t is types.DictType: | |
| 409 d = {} | |
| 410 for k,v in ao.items(): | |
| 411 kvd = crefutil._DictKeyAndValue(d) | |
| 412 self.unjellyInto(kvd, 0, k) | |
| 413 self.unjellyInto(kvd, 1, v) | |
| 414 return d | |
| 415 | |
| 416 else: | |
| 417 raise TypeError("Unsupported AOT type: %s" % t) | |
| 418 | |
| 419 del self.stack[-1] | |
| 420 | |
| 421 | |
| 422 def unjelly(self, ao): | |
| 423 try: | |
| 424 l = [None] | |
| 425 self.unjellyInto(l, 0, ao) | |
| 426 for callable, v in self.afterUnjelly: | |
| 427 callable(v[0]) | |
| 428 return l[0] | |
| 429 except: | |
| 430 log.msg("Error jellying object! Stacktrace follows::") | |
| 431 log.msg(string.join(map(repr, self.stack), "\n")) | |
| 432 raise | |
| 433 ######### | |
| 434 # Jelly # | |
| 435 ######### | |
| 436 | |
| 437 | |
| 438 def jellyToAOT(obj): | |
| 439 """Convert an object to an Abstract Object Tree.""" | |
| 440 return AOTJellier().jelly(obj) | |
| 441 | |
| 442 def jellyToSource(obj, file=None): | |
| 443 """ | |
| 444 Pass me an object and, optionally, a file object. | |
| 445 I'll convert the object to an AOT either return it (if no file was | |
| 446 specified) or write it to the file. | |
| 447 """ | |
| 448 | |
| 449 aot = jellyToAOT(obj) | |
| 450 if file: | |
| 451 file.write(getSource(aot)) | |
| 452 else: | |
| 453 return getSource(aot) | |
| 454 | |
| 455 | |
| 456 class AOTJellier: | |
| 457 def __init__(self): | |
| 458 # dict of {id(obj): (obj, node)} | |
| 459 self.prepared = {} | |
| 460 self._ref_id = 0 | |
| 461 self.stack = [] | |
| 462 | |
| 463 def prepareForRef(self, aoref, object): | |
| 464 """I prepare an object for later referencing, by storing its id() | |
| 465 and its _AORef in a cache.""" | |
| 466 self.prepared[id(object)] = aoref | |
| 467 | |
| 468 def jellyToAO(self, obj): | |
| 469 """I turn an object into an AOT and return it.""" | |
| 470 objType = type(obj) | |
| 471 self.stack.append(repr(obj)) | |
| 472 | |
| 473 #immutable: We don't care if these have multiple refs! | |
| 474 if objType in _SIMPLE_BUILTINS: | |
| 475 retval = obj | |
| 476 | |
| 477 elif objType is types.MethodType: | |
| 478 # TODO: make methods 'prefer' not to jelly the object internally, | |
| 479 # so that the object will show up where it's referenced first NOT | |
| 480 # by a method. | |
| 481 retval = InstanceMethod(obj.im_func.__name__, reflect.qual(obj.im_cl
ass), | |
| 482 self.jellyToAO(obj.im_self)) | |
| 483 | |
| 484 elif objType is types.ModuleType: | |
| 485 retval = Module(obj.__name__) | |
| 486 | |
| 487 elif objType is types.ClassType: | |
| 488 retval = Class(reflect.qual(obj)) | |
| 489 | |
| 490 elif issubclass(objType, type): | |
| 491 retval = Class(reflect.qual(obj)) | |
| 492 | |
| 493 elif objType is types.FunctionType: | |
| 494 retval = Function(reflect.fullFuncName(obj)) | |
| 495 | |
| 496 else: #mutable! gotta watch for refs. | |
| 497 | |
| 498 #Marmalade had the nicety of being able to just stick a 'reference' attribute | |
| 499 #on any Node object that was referenced, but in AOT, the referenced object | |
| 500 #is *inside* of a Ref call (Ref(num, obj) instead of | |
| 501 #<objtype ... reference="1">). The problem is, especially for built-in types, | |
| 502 #I can't just assign some attribute to them to give them a refnum. So, I have | |
| 503 #to "wrap" a Ref(..) around them later -- that's why I put *everything* that's | |
| 504 #mutable inside one. The Ref() class will only print the "Ref(..)" around an | |
| 505 #object if it has a Reference explicitly attached. | |
| 506 | |
| 507 if self.prepared.has_key(id(obj)): | |
| 508 oldRef = self.prepared[id(obj)] | |
| 509 if oldRef.refnum: | |
| 510 # it's been referenced already | |
| 511 key = oldRef.refnum | |
| 512 else: | |
| 513 # it hasn't been referenced yet | |
| 514 self._ref_id = self._ref_id + 1 | |
| 515 key = self._ref_id | |
| 516 oldRef.setRef(key) | |
| 517 return Deref(key) | |
| 518 | |
| 519 retval = Ref() | |
| 520 self.prepareForRef(retval, obj) | |
| 521 | |
| 522 if objType is types.ListType: | |
| 523 retval.setObj(map(self.jellyToAO, obj)) #hah! | |
| 524 | |
| 525 elif objType is types.TupleType: | |
| 526 retval.setObj(tuple(map(self.jellyToAO, obj))) | |
| 527 | |
| 528 elif objType is types.DictionaryType: | |
| 529 d = {} | |
| 530 for k,v in obj.items(): | |
| 531 d[self.jellyToAO(k)] = self.jellyToAO(v) | |
| 532 retval.setObj(d) | |
| 533 | |
| 534 elif objType is types.InstanceType: | |
| 535 if hasattr(obj, "__getstate__"): | |
| 536 state = self.jellyToAO(obj.__getstate__()) | |
| 537 else: | |
| 538 state = self.jellyToAO(obj.__dict__) | |
| 539 retval.setObj(Instance(reflect.qual(obj.__class__), state)) | |
| 540 | |
| 541 elif copy_reg.dispatch_table.has_key(objType): | |
| 542 unpickleFunc, state = copy_reg.dispatch_table[objType](obj) | |
| 543 | |
| 544 retval.setObj(Copyreg( reflect.fullFuncName(unpickleFunc), | |
| 545 self.jellyToAO(state))) | |
| 546 | |
| 547 else: | |
| 548 raise TypeError("Unsupported type: %s" % objType.__name__) | |
| 549 | |
| 550 del self.stack[-1] | |
| 551 return retval | |
| 552 | |
| 553 def jelly(self, obj): | |
| 554 try: | |
| 555 ao = self.jellyToAO(obj) | |
| 556 return ao | |
| 557 except: | |
| 558 log.msg("Error jellying object! Stacktrace follows::") | |
| 559 log.msg(string.join(self.stack, '\n')) | |
| 560 raise | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12261012:
Remove third_party/twisted_8_1 (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build