| OLD | NEW |
|---|
| (Empty) |
| 1 # Protocol Buffers - Google's data interchange format | |
| 2 # Copyright 2008 Google Inc. All rights reserved. | |
| 3 # http://code.google.com/p/protobuf/ | |
| 4 # | |
| 5 # Redistribution and use in source and binary forms, with or without | |
| 6 # modification, are permitted provided that the following conditions are | |
| 7 # met: | |
| 8 # | |
| 9 # * Redistributions of source code must retain the above copyright | |
| 10 # notice, this list of conditions and the following disclaimer. | |
| 11 # * Redistributions in binary form must reproduce the above | |
| 12 # copyright notice, this list of conditions and the following disclaimer | |
| 13 # in the documentation and/or other materials provided with the | |
| 14 # distribution. | |
| 15 # * Neither the name of Google Inc. nor the names of its | |
| 16 # contributors may be used to endorse or promote products derived from | |
| 17 # this software without specific prior written permission. | |
| 18 # | |
| 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 30 | |
| 31 #PY25 compatible for GAE. | |
| 32 # | |
| 33 # Copyright 2009 Google Inc. All Rights Reserved. | |
| 34 | |
| 35 """Code for decoding protocol buffer primitives. | |
| 36 | |
| 37 This code is very similar to encoder.py -- read the docs for that module first. | |
| 38 | |
| 39 A "decoder" is a function with the signature: | |
| 40 Decode(buffer, pos, end, message, field_dict) | |
| 41 The arguments are: | |
| 42 buffer: The string containing the encoded message. | |
| 43 pos: The current position in the string. | |
| 44 end: The position in the string where the current message ends. May be | |
| 45 less than len(buffer) if we're reading a sub-message. | |
| 46 message: The message object into which we're parsing. | |
| 47 field_dict: message._fields (avoids a hashtable lookup). | |
| 48 The decoder reads the field and stores it into field_dict, returning the new | |
| 49 buffer position. A decoder for a repeated field may proactively decode all of | |
| 50 the elements of that field, if they appear consecutively. | |
| 51 | |
| 52 Note that decoders may throw any of the following: | |
| 53 IndexError: Indicates a truncated message. | |
| 54 struct.error: Unpacking of a fixed-width field failed. | |
| 55 message.DecodeError: Other errors. | |
| 56 | |
| 57 Decoders are expected to raise an exception if they are called with pos > end. | |
| 58 This allows callers to be lax about bounds checking: it's fineto read past | |
| 59 "end" as long as you are sure that someone else will notice and throw an | |
| 60 exception later on. | |
| 61 | |
| 62 Something up the call stack is expected to catch IndexError and struct.error | |
| 63 and convert them to message.DecodeError. | |
| 64 | |
| 65 Decoders are constructed using decoder constructors with the signature: | |
| 66 MakeDecoder(field_number, is_repeated, is_packed, key, new_default) | |
| 67 The arguments are: | |
| 68 field_number: The field number of the field we want to decode. | |
| 69 is_repeated: Is the field a repeated field? (bool) | |
| 70 is_packed: Is the field a packed field? (bool) | |
| 71 key: The key to use when looking up the field within field_dict. | |
| 72 (This is actually the FieldDescriptor but nothing in this | |
| 73 file should depend on that.) | |
| 74 new_default: A function which takes a message object as a parameter and | |
| 75 returns a new instance of the default value for this field. | |
| 76 (This is called for repeated fields and sub-messages, when an | |
| 77 instance does not already exist.) | |
| 78 | |
| 79 As with encoders, we define a decoder constructor for every type of field. | |
| 80 Then, for every field of every message class we construct an actual decoder. | |
| 81 That decoder goes into a dict indexed by tag, so when we decode a message | |
| 82 we repeatedly read a tag, look up the corresponding decoder, and invoke it. | |
| 83 """ | |
| 84 | |
| 85 __author__ = 'kenton@google.com (Kenton Varda)' | |
| 86 | |
| 87 import struct | |
| 88 import sys ##PY25 | |
| 89 _PY2 = sys.version_info[0] < 3 ##PY25 | |
| 90 from google.protobuf.internal import encoder | |
| 91 from google.protobuf.internal import wire_format | |
| 92 from google.protobuf import message | |
| 93 | |
| 94 | |
| 95 # This will overflow and thus become IEEE-754 "infinity". We would use | |
| 96 # "float('inf')" but it doesn't work on Windows pre-Python-2.6. | |
| 97 _POS_INF = 1e10000 | |
| 98 _NEG_INF = -_POS_INF | |
| 99 _NAN = _POS_INF * 0 | |
| 100 | |
| 101 | |
| 102 # This is not for optimization, but rather to avoid conflicts with local | |
| 103 # variables named "message". | |
| 104 _DecodeError = message.DecodeError | |
| 105 | |
| 106 | |
| 107 def _VarintDecoder(mask, result_type): | |
| 108 """Return an encoder for a basic varint value (does not include tag). | |
| 109 | |
| 110 Decoded values will be bitwise-anded with the given mask before being | |
| 111 returned, e.g. to limit them to 32 bits. The returned decoder does not | |
| 112 take the usual "end" parameter -- the caller is expected to do bounds checking | |
| 113 after the fact (often the caller can defer such checking until later). The | |
| 114 decoder returns a (value, new_pos) pair. | |
| 115 """ | |
| 116 | |
| 117 local_ord = ord | |
| 118 py2 = _PY2 ##PY25 | |
| 119 ##!PY25 py2 = str is bytes | |
| 120 def DecodeVarint(buffer, pos): | |
| 121 result = 0 | |
| 122 shift = 0 | |
| 123 while 1: | |
| 124 b = local_ord(buffer[pos]) if py2 else buffer[pos] | |
| 125 result |= ((b & 0x7f) << shift) | |
| 126 pos += 1 | |
| 127 if not (b & 0x80): | |
| 128 result &= mask | |
| 129 result = result_type(result) | |
| 130 return (result, pos) | |
| 131 shift += 7 | |
| 132 if shift >= 64: | |
| 133 raise _DecodeError('Too many bytes when decoding varint.') | |
| 134 return DecodeVarint | |
| 135 | |
| 136 | |
| 137 def _SignedVarintDecoder(mask, result_type): | |
| 138 """Like _VarintDecoder() but decodes signed values.""" | |
| 139 | |
| 140 local_ord = ord | |
| 141 py2 = _PY2 ##PY25 | |
| 142 ##!PY25 py2 = str is bytes | |
| 143 def DecodeVarint(buffer, pos): | |
| 144 result = 0 | |
| 145 shift = 0 | |
| 146 while 1: | |
| 147 b = local_ord(buffer[pos]) if py2 else buffer[pos] | |
| 148 result |= ((b & 0x7f) << shift) | |
| 149 pos += 1 | |
| 150 if not (b & 0x80): | |
| 151 if result > 0x7fffffffffffffff: | |
| 152 result -= (1 << 64) | |
| 153 result |= ~mask | |
| 154 else: | |
| 155 result &= mask | |
| 156 result = result_type(result) | |
| 157 return (result, pos) | |
| 158 shift += 7 | |
| 159 if shift >= 64: | |
| 160 raise _DecodeError('Too many bytes when decoding varint.') | |
| 161 return DecodeVarint | |
| 162 | |
| 163 # We force 32-bit values to int and 64-bit values to long to make | |
| 164 # alternate implementations where the distinction is more significant | |
| 165 # (e.g. the C++ implementation) simpler. | |
| 166 | |
| 167 _DecodeVarint = _VarintDecoder((1 << 64) - 1, long) | |
| 168 _DecodeSignedVarint = _SignedVarintDecoder((1 << 64) - 1, long) | |
| 169 | |
| 170 # Use these versions for values which must be limited to 32 bits. | |
| 171 _DecodeVarint32 = _VarintDecoder((1 << 32) - 1, int) | |
| 172 _DecodeSignedVarint32 = _SignedVarintDecoder((1 << 32) - 1, int) | |
| 173 | |
| 174 | |
| 175 def ReadTag(buffer, pos): | |
| 176 """Read a tag from the buffer, and return a (tag_bytes, new_pos) tuple. | |
| 177 | |
| 178 We return the raw bytes of the tag rather than decoding them. The raw | |
| 179 bytes can then be used to look up the proper decoder. This effectively allows | |
| 180 us to trade some work that would be done in pure-python (decoding a varint) | |
| 181 for work that is done in C (searching for a byte string in a hash table). | |
| 182 In a low-level language it would be much cheaper to decode the varint and | |
| 183 use that, but not in Python. | |
| 184 """ | |
| 185 | |
| 186 py2 = _PY2 ##PY25 | |
| 187 ##!PY25 py2 = str is bytes | |
| 188 start = pos | |
| 189 while (ord(buffer[pos]) if py2 else buffer[pos]) & 0x80: | |
| 190 pos += 1 | |
| 191 pos += 1 | |
| 192 return (buffer[start:pos], pos) | |
| 193 | |
| 194 | |
| 195 # -------------------------------------------------------------------- | |
| 196 | |
| 197 | |
| 198 def _SimpleDecoder(wire_type, decode_value): | |
| 199 """Return a constructor for a decoder for fields of a particular type. | |
| 200 | |
| 201 Args: | |
| 202 wire_type: The field's wire type. | |
| 203 decode_value: A function which decodes an individual value, e.g. | |
| 204 _DecodeVarint() | |
| 205 """ | |
| 206 | |
| 207 def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default): | |
| 208 if is_packed: | |
| 209 local_DecodeVarint = _DecodeVarint | |
| 210 def DecodePackedField(buffer, pos, end, message, field_dict): | |
| 211 value = field_dict.get(key) | |
| 212 if value is None: | |
| 213 value = field_dict.setdefault(key, new_default(message)) | |
| 214 (endpoint, pos) = local_DecodeVarint(buffer, pos) | |
| 215 endpoint += pos | |
| 216 if endpoint > end: | |
| 217 raise _DecodeError('Truncated message.') | |
| 218 while pos < endpoint: | |
| 219 (element, pos) = decode_value(buffer, pos) | |
| 220 value.append(element) | |
| 221 if pos > endpoint: | |
| 222 del value[-1] # Discard corrupt value. | |
| 223 raise _DecodeError('Packed element was truncated.') | |
| 224 return pos | |
| 225 return DecodePackedField | |
| 226 elif is_repeated: | |
| 227 tag_bytes = encoder.TagBytes(field_number, wire_type) | |
| 228 tag_len = len(tag_bytes) | |
| 229 def DecodeRepeatedField(buffer, pos, end, message, field_dict): | |
| 230 value = field_dict.get(key) | |
| 231 if value is None: | |
| 232 value = field_dict.setdefault(key, new_default(message)) | |
| 233 while 1: | |
| 234 (element, new_pos) = decode_value(buffer, pos) | |
| 235 value.append(element) | |
| 236 # Predict that the next tag is another copy of the same repeated | |
| 237 # field. | |
| 238 pos = new_pos + tag_len | |
| 239 if buffer[new_pos:pos] != tag_bytes or new_pos >= end: | |
| 240 # Prediction failed. Return. | |
| 241 if new_pos > end: | |
| 242 raise _DecodeError('Truncated message.') | |
| 243 return new_pos | |
| 244 return DecodeRepeatedField | |
| 245 else: | |
| 246 def DecodeField(buffer, pos, end, message, field_dict): | |
| 247 (field_dict[key], pos) = decode_value(buffer, pos) | |
| 248 if pos > end: | |
| 249 del field_dict[key] # Discard corrupt value. | |
| 250 raise _DecodeError('Truncated message.') | |
| 251 return pos | |
| 252 return DecodeField | |
| 253 | |
| 254 return SpecificDecoder | |
| 255 | |
| 256 | |
| 257 def _ModifiedDecoder(wire_type, decode_value, modify_value): | |
| 258 """Like SimpleDecoder but additionally invokes modify_value on every value | |
| 259 before storing it. Usually modify_value is ZigZagDecode. | |
| 260 """ | |
| 261 | |
| 262 # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but | |
| 263 # not enough to make a significant difference. | |
| 264 | |
| 265 def InnerDecode(buffer, pos): | |
| 266 (result, new_pos) = decode_value(buffer, pos) | |
| 267 return (modify_value(result), new_pos) | |
| 268 return _SimpleDecoder(wire_type, InnerDecode) | |
| 269 | |
| 270 | |
| 271 def _StructPackDecoder(wire_type, format): | |
| 272 """Return a constructor for a decoder for a fixed-width field. | |
| 273 | |
| 274 Args: | |
| 275 wire_type: The field's wire type. | |
| 276 format: The format string to pass to struct.unpack(). | |
| 277 """ | |
| 278 | |
| 279 value_size = struct.calcsize(format) | |
| 280 local_unpack = struct.unpack | |
| 281 | |
| 282 # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but | |
| 283 # not enough to make a significant difference. | |
| 284 | |
| 285 # Note that we expect someone up-stack to catch struct.error and convert | |
| 286 # it to _DecodeError -- this way we don't have to set up exception- | |
| 287 # handling blocks every time we parse one value. | |
| 288 | |
| 289 def InnerDecode(buffer, pos): | |
| 290 new_pos = pos + value_size | |
| 291 result = local_unpack(format, buffer[pos:new_pos])[0] | |
| 292 return (result, new_pos) | |
| 293 return _SimpleDecoder(wire_type, InnerDecode) | |
| 294 | |
| 295 | |
| 296 def _FloatDecoder(): | |
| 297 """Returns a decoder for a float field. | |
| 298 | |
| 299 This code works around a bug in struct.unpack for non-finite 32-bit | |
| 300 floating-point values. | |
| 301 """ | |
| 302 | |
| 303 local_unpack = struct.unpack | |
| 304 b = (lambda x:x) if _PY2 else lambda x:x.encode('latin1') ##PY25 | |
| 305 | |
| 306 def InnerDecode(buffer, pos): | |
| 307 # We expect a 32-bit value in little-endian byte order. Bit 1 is the sign | |
| 308 # bit, bits 2-9 represent the exponent, and bits 10-32 are the significand. | |
| 309 new_pos = pos + 4 | |
| 310 float_bytes = buffer[pos:new_pos] | |
| 311 | |
| 312 # If this value has all its exponent bits set, then it's non-finite. | |
| 313 # In Python 2.4, struct.unpack will convert it to a finite 64-bit value. | |
| 314 # To avoid that, we parse it specially. | |
| 315 if ((float_bytes[3:4] in b('\x7F\xFF')) ##PY25 | |
| 316 ##!PY25 if ((float_bytes[3:4] in b'\x7F\xFF') | |
| 317 and (float_bytes[2:3] >= b('\x80'))): ##PY25 | |
| 318 ##!PY25 and (float_bytes[2:3] >= b'\x80')): | |
| 319 # If at least one significand bit is set... | |
| 320 if float_bytes[0:3] != b('\x00\x00\x80'): ##PY25 | |
| 321 ##!PY25 if float_bytes[0:3] != b'\x00\x00\x80': | |
| 322 return (_NAN, new_pos) | |
| 323 # If sign bit is set... | |
| 324 if float_bytes[3:4] == b('\xFF'): ##PY25 | |
| 325 ##!PY25 if float_bytes[3:4] == b'\xFF': | |
| 326 return (_NEG_INF, new_pos) | |
| 327 return (_POS_INF, new_pos) | |
| 328 | |
| 329 # Note that we expect someone up-stack to catch struct.error and convert | |
| 330 # it to _DecodeError -- this way we don't have to set up exception- | |
| 331 # handling blocks every time we parse one value. | |
| 332 result = local_unpack('<f', float_bytes)[0] | |
| 333 return (result, new_pos) | |
| 334 return _SimpleDecoder(wire_format.WIRETYPE_FIXED32, InnerDecode) | |
| 335 | |
| 336 | |
| 337 def _DoubleDecoder(): | |
| 338 """Returns a decoder for a double field. | |
| 339 | |
| 340 This code works around a bug in struct.unpack for not-a-number. | |
| 341 """ | |
| 342 | |
| 343 local_unpack = struct.unpack | |
| 344 b = (lambda x:x) if _PY2 else lambda x:x.encode('latin1') ##PY25 | |
| 345 | |
| 346 def InnerDecode(buffer, pos): | |
| 347 # We expect a 64-bit value in little-endian byte order. Bit 1 is the sign | |
| 348 # bit, bits 2-12 represent the exponent, and bits 13-64 are the significand. | |
| 349 new_pos = pos + 8 | |
| 350 double_bytes = buffer[pos:new_pos] | |
| 351 | |
| 352 # If this value has all its exponent bits set and at least one significand | |
| 353 # bit set, it's not a number. In Python 2.4, struct.unpack will treat it | |
| 354 # as inf or -inf. To avoid that, we treat it specially. | |
| 355 ##!PY25 if ((double_bytes[7:8] in b'\x7F\xFF') | |
| 356 ##!PY25 and (double_bytes[6:7] >= b'\xF0') | |
| 357 ##!PY25 and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')): | |
| 358 if ((double_bytes[7:8] in b('\x7F\xFF')) ##PY25 | |
| 359 and (double_bytes[6:7] >= b('\xF0')) ##PY25 | |
| 360 and (double_bytes[0:7] != b('\x00\x00\x00\x00\x00\x00\xF0'))): ##PY25 | |
| 361 return (_NAN, new_pos) | |
| 362 | |
| 363 # Note that we expect someone up-stack to catch struct.error and convert | |
| 364 # it to _DecodeError -- this way we don't have to set up exception- | |
| 365 # handling blocks every time we parse one value. | |
| 366 result = local_unpack('<d', double_bytes)[0] | |
| 367 return (result, new_pos) | |
| 368 return _SimpleDecoder(wire_format.WIRETYPE_FIXED64, InnerDecode) | |
| 369 | |
| 370 | |
| 371 def EnumDecoder(field_number, is_repeated, is_packed, key, new_default): | |
| 372 enum_type = key.enum_type | |
| 373 if is_packed: | |
| 374 local_DecodeVarint = _DecodeVarint | |
| 375 def DecodePackedField(buffer, pos, end, message, field_dict): | |
| 376 value = field_dict.get(key) | |
| 377 if value is None: | |
| 378 value = field_dict.setdefault(key, new_default(message)) | |
| 379 (endpoint, pos) = local_DecodeVarint(buffer, pos) | |
| 380 endpoint += pos | |
| 381 if endpoint > end: | |
| 382 raise _DecodeError('Truncated message.') | |
| 383 while pos < endpoint: | |
| 384 value_start_pos = pos | |
| 385 (element, pos) = _DecodeSignedVarint32(buffer, pos) | |
| 386 if element in enum_type.values_by_number: | |
| 387 value.append(element) | |
| 388 else: | |
| 389 if not message._unknown_fields: | |
| 390 message._unknown_fields = [] | |
| 391 tag_bytes = encoder.TagBytes(field_number, | |
| 392 wire_format.WIRETYPE_VARINT) | |
| 393 message._unknown_fields.append( | |
| 394 (tag_bytes, buffer[value_start_pos:pos])) | |
| 395 if pos > endpoint: | |
| 396 if element in enum_type.values_by_number: | |
| 397 del value[-1] # Discard corrupt value. | |
| 398 else: | |
| 399 del message._unknown_fields[-1] | |
| 400 raise _DecodeError('Packed element was truncated.') | |
| 401 return pos | |
| 402 return DecodePackedField | |
| 403 elif is_repeated: | |
| 404 tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_VARINT) | |
| 405 tag_len = len(tag_bytes) | |
| 406 def DecodeRepeatedField(buffer, pos, end, message, field_dict): | |
| 407 value = field_dict.get(key) | |
| 408 if value is None: | |
| 409 value = field_dict.setdefault(key, new_default(message)) | |
| 410 while 1: | |
| 411 (element, new_pos) = _DecodeSignedVarint32(buffer, pos) | |
| 412 if element in enum_type.values_by_number: | |
| 413 value.append(element) | |
| 414 else: | |
| 415 if not message._unknown_fields: | |
| 416 message._unknown_fields = [] | |
| 417 message._unknown_fields.append( | |
| 418 (tag_bytes, buffer[pos:new_pos])) | |
| 419 # Predict that the next tag is another copy of the same repeated | |
| 420 # field. | |
| 421 pos = new_pos + tag_len | |
| 422 if buffer[new_pos:pos] != tag_bytes or new_pos >= end: | |
| 423 # Prediction failed. Return. | |
| 424 if new_pos > end: | |
| 425 raise _DecodeError('Truncated message.') | |
| 426 return new_pos | |
| 427 return DecodeRepeatedField | |
| 428 else: | |
| 429 def DecodeField(buffer, pos, end, message, field_dict): | |
| 430 value_start_pos = pos | |
| 431 (enum_value, pos) = _DecodeSignedVarint32(buffer, pos) | |
| 432 if pos > end: | |
| 433 raise _DecodeError('Truncated message.') | |
| 434 if enum_value in enum_type.values_by_number: | |
| 435 field_dict[key] = enum_value | |
| 436 else: | |
| 437 if not message._unknown_fields: | |
| 438 message._unknown_fields = [] | |
| 439 tag_bytes = encoder.TagBytes(field_number, | |
| 440 wire_format.WIRETYPE_VARINT) | |
| 441 message._unknown_fields.append( | |
| 442 (tag_bytes, buffer[value_start_pos:pos])) | |
| 443 return pos | |
| 444 return DecodeField | |
| 445 | |
| 446 | |
| 447 # -------------------------------------------------------------------- | |
| 448 | |
| 449 | |
| 450 Int32Decoder = _SimpleDecoder( | |
| 451 wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32) | |
| 452 | |
| 453 Int64Decoder = _SimpleDecoder( | |
| 454 wire_format.WIRETYPE_VARINT, _DecodeSignedVarint) | |
| 455 | |
| 456 UInt32Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint32) | |
| 457 UInt64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint) | |
| 458 | |
| 459 SInt32Decoder = _ModifiedDecoder( | |
| 460 wire_format.WIRETYPE_VARINT, _DecodeVarint32, wire_format.ZigZagDecode) | |
| 461 SInt64Decoder = _ModifiedDecoder( | |
| 462 wire_format.WIRETYPE_VARINT, _DecodeVarint, wire_format.ZigZagDecode) | |
| 463 | |
| 464 # Note that Python conveniently guarantees that when using the '<' prefix on | |
| 465 # formats, they will also have the same size across all platforms (as opposed | |
| 466 # to without the prefix, where their sizes depend on the C compiler's basic | |
| 467 # type sizes). | |
| 468 Fixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<I') | |
| 469 Fixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<Q') | |
| 470 SFixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<i') | |
| 471 SFixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<q') | |
| 472 FloatDecoder = _FloatDecoder() | |
| 473 DoubleDecoder = _DoubleDecoder() | |
| 474 | |
| 475 BoolDecoder = _ModifiedDecoder( | |
| 476 wire_format.WIRETYPE_VARINT, _DecodeVarint, bool) | |
| 477 | |
| 478 | |
| 479 def StringDecoder(field_number, is_repeated, is_packed, key, new_default): | |
| 480 """Returns a decoder for a string field.""" | |
| 481 | |
| 482 local_DecodeVarint = _DecodeVarint | |
| 483 local_unicode = unicode | |
| 484 | |
| 485 def _ConvertToUnicode(byte_str): | |
| 486 try: | |
| 487 return local_unicode(byte_str, 'utf-8') | |
| 488 except UnicodeDecodeError, e: | |
| 489 # add more information to the error message and re-raise it. | |
| 490 e.reason = '%s in field: %s' % (e, key.full_name) | |
| 491 raise | |
| 492 | |
| 493 assert not is_packed | |
| 494 if is_repeated: | |
| 495 tag_bytes = encoder.TagBytes(field_number, | |
| 496 wire_format.WIRETYPE_LENGTH_DELIMITED) | |
| 497 tag_len = len(tag_bytes) | |
| 498 def DecodeRepeatedField(buffer, pos, end, message, field_dict): | |
| 499 value = field_dict.get(key) | |
| 500 if value is None: | |
| 501 value = field_dict.setdefault(key, new_default(message)) | |
| 502 while 1: | |
| 503 (size, pos) = local_DecodeVarint(buffer, pos) | |
| 504 new_pos = pos + size | |
| 505 if new_pos > end: | |
| 506 raise _DecodeError('Truncated string.') | |
| 507 value.append(_ConvertToUnicode(buffer[pos:new_pos])) | |
| 508 # Predict that the next tag is another copy of the same repeated field. | |
| 509 pos = new_pos + tag_len | |
| 510 if buffer[new_pos:pos] != tag_bytes or new_pos == end: | |
| 511 # Prediction failed. Return. | |
| 512 return new_pos | |
| 513 return DecodeRepeatedField | |
| 514 else: | |
| 515 def DecodeField(buffer, pos, end, message, field_dict): | |
| 516 (size, pos) = local_DecodeVarint(buffer, pos) | |
| 517 new_pos = pos + size | |
| 518 if new_pos > end: | |
| 519 raise _DecodeError('Truncated string.') | |
| 520 field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos]) | |
| 521 return new_pos | |
| 522 return DecodeField | |
| 523 | |
| 524 | |
| 525 def BytesDecoder(field_number, is_repeated, is_packed, key, new_default): | |
| 526 """Returns a decoder for a bytes field.""" | |
| 527 | |
| 528 local_DecodeVarint = _DecodeVarint | |
| 529 | |
| 530 assert not is_packed | |
| 531 if is_repeated: | |
| 532 tag_bytes = encoder.TagBytes(field_number, | |
| 533 wire_format.WIRETYPE_LENGTH_DELIMITED) | |
| 534 tag_len = len(tag_bytes) | |
| 535 def DecodeRepeatedField(buffer, pos, end, message, field_dict): | |
| 536 value = field_dict.get(key) | |
| 537 if value is None: | |
| 538 value = field_dict.setdefault(key, new_default(message)) | |
| 539 while 1: | |
| 540 (size, pos) = local_DecodeVarint(buffer, pos) | |
| 541 new_pos = pos + size | |
| 542 if new_pos > end: | |
| 543 raise _DecodeError('Truncated string.') | |
| 544 value.append(buffer[pos:new_pos]) | |
| 545 # Predict that the next tag is another copy of the same repeated field. | |
| 546 pos = new_pos + tag_len | |
| 547 if buffer[new_pos:pos] != tag_bytes or new_pos == end: | |
| 548 # Prediction failed. Return. | |
| 549 return new_pos | |
| 550 return DecodeRepeatedField | |
| 551 else: | |
| 552 def DecodeField(buffer, pos, end, message, field_dict): | |
| 553 (size, pos) = local_DecodeVarint(buffer, pos) | |
| 554 new_pos = pos + size | |
| 555 if new_pos > end: | |
| 556 raise _DecodeError('Truncated string.') | |
| 557 field_dict[key] = buffer[pos:new_pos] | |
| 558 return new_pos | |
| 559 return DecodeField | |
| 560 | |
| 561 | |
| 562 def GroupDecoder(field_number, is_repeated, is_packed, key, new_default): | |
| 563 """Returns a decoder for a group field.""" | |
| 564 | |
| 565 end_tag_bytes = encoder.TagBytes(field_number, | |
| 566 wire_format.WIRETYPE_END_GROUP) | |
| 567 end_tag_len = len(end_tag_bytes) | |
| 568 | |
| 569 assert not is_packed | |
| 570 if is_repeated: | |
| 571 tag_bytes = encoder.TagBytes(field_number, | |
| 572 wire_format.WIRETYPE_START_GROUP) | |
| 573 tag_len = len(tag_bytes) | |
| 574 def DecodeRepeatedField(buffer, pos, end, message, field_dict): | |
| 575 value = field_dict.get(key) | |
| 576 if value is None: | |
| 577 value = field_dict.setdefault(key, new_default(message)) | |
| 578 while 1: | |
| 579 value = field_dict.get(key) | |
| 580 if value is None: | |
| 581 value = field_dict.setdefault(key, new_default(message)) | |
| 582 # Read sub-message. | |
| 583 pos = value.add()._InternalParse(buffer, pos, end) | |
| 584 # Read end tag. | |
| 585 new_pos = pos+end_tag_len | |
| 586 if buffer[pos:new_pos] != end_tag_bytes or new_pos > end: | |
| 587 raise _DecodeError('Missing group end tag.') | |
| 588 # Predict that the next tag is another copy of the same repeated field. | |
| 589 pos = new_pos + tag_len | |
| 590 if buffer[new_pos:pos] != tag_bytes or new_pos == end: | |
| 591 # Prediction failed. Return. | |
| 592 return new_pos | |
| 593 return DecodeRepeatedField | |
| 594 else: | |
| 595 def DecodeField(buffer, pos, end, message, field_dict): | |
| 596 value = field_dict.get(key) | |
| 597 if value is None: | |
| 598 value = field_dict.setdefault(key, new_default(message)) | |
| 599 # Read sub-message. | |
| 600 pos = value._InternalParse(buffer, pos, end) | |
| 601 # Read end tag. | |
| 602 new_pos = pos+end_tag_len | |
| 603 if buffer[pos:new_pos] != end_tag_bytes or new_pos > end: | |
| 604 raise _DecodeError('Missing group end tag.') | |
| 605 return new_pos | |
| 606 return DecodeField | |
| 607 | |
| 608 | |
| 609 def MessageDecoder(field_number, is_repeated, is_packed, key, new_default): | |
| 610 """Returns a decoder for a message field.""" | |
| 611 | |
| 612 local_DecodeVarint = _DecodeVarint | |
| 613 | |
| 614 assert not is_packed | |
| 615 if is_repeated: | |
| 616 tag_bytes = encoder.TagBytes(field_number, | |
| 617 wire_format.WIRETYPE_LENGTH_DELIMITED) | |
| 618 tag_len = len(tag_bytes) | |
| 619 def DecodeRepeatedField(buffer, pos, end, message, field_dict): | |
| 620 value = field_dict.get(key) | |
| 621 if value is None: | |
| 622 value = field_dict.setdefault(key, new_default(message)) | |
| 623 while 1: | |
| 624 value = field_dict.get(key) | |
| 625 if value is None: | |
| 626 value = field_dict.setdefault(key, new_default(message)) | |
| 627 # Read length. | |
| 628 (size, pos) = local_DecodeVarint(buffer, pos) | |
| 629 new_pos = pos + size | |
| 630 if new_pos > end: | |
| 631 raise _DecodeError('Truncated message.') | |
| 632 # Read sub-message. | |
| 633 if value.add()._InternalParse(buffer, pos, new_pos) != new_pos: | |
| 634 # The only reason _InternalParse would return early is if it | |
| 635 # encountered an end-group tag. | |
| 636 raise _DecodeError('Unexpected end-group tag.') | |
| 637 # Predict that the next tag is another copy of the same repeated field. | |
| 638 pos = new_pos + tag_len | |
| 639 if buffer[new_pos:pos] != tag_bytes or new_pos == end: | |
| 640 # Prediction failed. Return. | |
| 641 return new_pos | |
| 642 return DecodeRepeatedField | |
| 643 else: | |
| 644 def DecodeField(buffer, pos, end, message, field_dict): | |
| 645 value = field_dict.get(key) | |
| 646 if value is None: | |
| 647 value = field_dict.setdefault(key, new_default(message)) | |
| 648 # Read length. | |
| 649 (size, pos) = local_DecodeVarint(buffer, pos) | |
| 650 new_pos = pos + size | |
| 651 if new_pos > end: | |
| 652 raise _DecodeError('Truncated message.') | |
| 653 # Read sub-message. | |
| 654 if value._InternalParse(buffer, pos, new_pos) != new_pos: | |
| 655 # The only reason _InternalParse would return early is if it encountered | |
| 656 # an end-group tag. | |
| 657 raise _DecodeError('Unexpected end-group tag.') | |
| 658 return new_pos | |
| 659 return DecodeField | |
| 660 | |
| 661 | |
| 662 # -------------------------------------------------------------------- | |
| 663 | |
| 664 MESSAGE_SET_ITEM_TAG = encoder.TagBytes(1, wire_format.WIRETYPE_START_GROUP) | |
| 665 | |
| 666 def MessageSetItemDecoder(extensions_by_number): | |
| 667 """Returns a decoder for a MessageSet item. | |
| 668 | |
| 669 The parameter is the _extensions_by_number map for the message class. | |
| 670 | |
| 671 The message set message looks like this: | |
| 672 message MessageSet { | |
| 673 repeated group Item = 1 { | |
| 674 required int32 type_id = 2; | |
| 675 required string message = 3; | |
| 676 } | |
| 677 } | |
| 678 """ | |
| 679 | |
| 680 type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT) | |
| 681 message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED) | |
| 682 item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP) | |
| 683 | |
| 684 local_ReadTag = ReadTag | |
| 685 local_DecodeVarint = _DecodeVarint | |
| 686 local_SkipField = SkipField | |
| 687 | |
| 688 def DecodeItem(buffer, pos, end, message, field_dict): | |
| 689 message_set_item_start = pos | |
| 690 type_id = -1 | |
| 691 message_start = -1 | |
| 692 message_end = -1 | |
| 693 | |
| 694 # Technically, type_id and message can appear in any order, so we need | |
| 695 # a little loop here. | |
| 696 while 1: | |
| 697 (tag_bytes, pos) = local_ReadTag(buffer, pos) | |
| 698 if tag_bytes == type_id_tag_bytes: | |
| 699 (type_id, pos) = local_DecodeVarint(buffer, pos) | |
| 700 elif tag_bytes == message_tag_bytes: | |
| 701 (size, message_start) = local_DecodeVarint(buffer, pos) | |
| 702 pos = message_end = message_start + size | |
| 703 elif tag_bytes == item_end_tag_bytes: | |
| 704 break | |
| 705 else: | |
| 706 pos = SkipField(buffer, pos, end, tag_bytes) | |
| 707 if pos == -1: | |
| 708 raise _DecodeError('Missing group end tag.') | |
| 709 | |
| 710 if pos > end: | |
| 711 raise _DecodeError('Truncated message.') | |
| 712 | |
| 713 if type_id == -1: | |
| 714 raise _DecodeError('MessageSet item missing type_id.') | |
| 715 if message_start == -1: | |
| 716 raise _DecodeError('MessageSet item missing message.') | |
| 717 | |
| 718 extension = extensions_by_number.get(type_id) | |
| 719 if extension is not None: | |
| 720 value = field_dict.get(extension) | |
| 721 if value is None: | |
| 722 value = field_dict.setdefault( | |
| 723 extension, extension.message_type._concrete_class()) | |
| 724 if value._InternalParse(buffer, message_start,message_end) != message_end: | |
| 725 # The only reason _InternalParse would return early is if it encountered | |
| 726 # an end-group tag. | |
| 727 raise _DecodeError('Unexpected end-group tag.') | |
| 728 else: | |
| 729 if not message._unknown_fields: | |
| 730 message._unknown_fields = [] | |
| 731 message._unknown_fields.append((MESSAGE_SET_ITEM_TAG, | |
| 732 buffer[message_set_item_start:pos])) | |
| 733 | |
| 734 return pos | |
| 735 | |
| 736 return DecodeItem | |
| 737 | |
| 738 # -------------------------------------------------------------------- | |
| 739 # Optimization is not as heavy here because calls to SkipField() are rare, | |
| 740 # except for handling end-group tags. | |
| 741 | |
| 742 def _SkipVarint(buffer, pos, end): | |
| 743 """Skip a varint value. Returns the new position.""" | |
| 744 # Previously ord(buffer[pos]) raised IndexError when pos is out of range. | |
| 745 # With this code, ord(b'') raises TypeError. Both are handled in | |
| 746 # python_message.py to generate a 'Truncated message' error. | |
| 747 while ord(buffer[pos:pos+1]) & 0x80: | |
| 748 pos += 1 | |
| 749 pos += 1 | |
| 750 if pos > end: | |
| 751 raise _DecodeError('Truncated message.') | |
| 752 return pos | |
| 753 | |
| 754 def _SkipFixed64(buffer, pos, end): | |
| 755 """Skip a fixed64 value. Returns the new position.""" | |
| 756 | |
| 757 pos += 8 | |
| 758 if pos > end: | |
| 759 raise _DecodeError('Truncated message.') | |
| 760 return pos | |
| 761 | |
| 762 def _SkipLengthDelimited(buffer, pos, end): | |
| 763 """Skip a length-delimited value. Returns the new position.""" | |
| 764 | |
| 765 (size, pos) = _DecodeVarint(buffer, pos) | |
| 766 pos += size | |
| 767 if pos > end: | |
| 768 raise _DecodeError('Truncated message.') | |
| 769 return pos | |
| 770 | |
| 771 def _SkipGroup(buffer, pos, end): | |
| 772 """Skip sub-group. Returns the new position.""" | |
| 773 | |
| 774 while 1: | |
| 775 (tag_bytes, pos) = ReadTag(buffer, pos) | |
| 776 new_pos = SkipField(buffer, pos, end, tag_bytes) | |
| 777 if new_pos == -1: | |
| 778 return pos | |
| 779 pos = new_pos | |
| 780 | |
| 781 def _EndGroup(buffer, pos, end): | |
| 782 """Skipping an END_GROUP tag returns -1 to tell the parent loop to break.""" | |
| 783 | |
| 784 return -1 | |
| 785 | |
| 786 def _SkipFixed32(buffer, pos, end): | |
| 787 """Skip a fixed32 value. Returns the new position.""" | |
| 788 | |
| 789 pos += 4 | |
| 790 if pos > end: | |
| 791 raise _DecodeError('Truncated message.') | |
| 792 return pos | |
| 793 | |
| 794 def _RaiseInvalidWireType(buffer, pos, end): | |
| 795 """Skip function for unknown wire types. Raises an exception.""" | |
| 796 | |
| 797 raise _DecodeError('Tag had invalid wire type.') | |
| 798 | |
| 799 def _FieldSkipper(): | |
| 800 """Constructs the SkipField function.""" | |
| 801 | |
| 802 WIRETYPE_TO_SKIPPER = [ | |
| 803 _SkipVarint, | |
| 804 _SkipFixed64, | |
| 805 _SkipLengthDelimited, | |
| 806 _SkipGroup, | |
| 807 _EndGroup, | |
| 808 _SkipFixed32, | |
| 809 _RaiseInvalidWireType, | |
| 810 _RaiseInvalidWireType, | |
| 811 ] | |
| 812 | |
| 813 wiretype_mask = wire_format.TAG_TYPE_MASK | |
| 814 | |
| 815 def SkipField(buffer, pos, end, tag_bytes): | |
| 816 """Skips a field with the specified tag. | |
| 817 | |
| 818 |pos| should point to the byte immediately after the tag. | |
| 819 | |
| 820 Returns: | |
| 821 The new position (after the tag value), or -1 if the tag is an end-group | |
| 822 tag (in which case the calling loop should break). | |
| 823 """ | |
| 824 | |
| 825 # The wire type is always in the first byte since varints are little-endian. | |
| 826 wire_type = ord(tag_bytes[0:1]) & wiretype_mask | |
| 827 return WIRETYPE_TO_SKIPPER[wire_type](buffer, pos, end) | |
| 828 | |
| 829 return SkipField | |
| 830 | |
| 831 SkipField = _FieldSkipper() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1162993005:
Renamed google.protobuf to protobuf26 (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools