| OLD | NEW |
| (Empty) |
| 1 # -*- coding: utf-8 -*- | |
| 2 # | |
| 3 # Copyright 2011 Sybren A. Stüvel <sybren@stuvel.eu> | |
| 4 # | |
| 5 # Licensed under the Apache License, Version 2.0 (the "License"); | |
| 6 # you may not use this file except in compliance with the License. | |
| 7 # You may obtain a copy of the License at | |
| 8 # | |
| 9 # http://www.apache.org/licenses/LICENSE-2.0 | |
| 10 # | |
| 11 # Unless required by applicable law or agreed to in writing, software | |
| 12 # distributed under the License is distributed on an "AS IS" BASIS, | |
| 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 14 # See the License for the specific language governing permissions and | |
| 15 # limitations under the License. | |
| 16 | |
| 17 '''VARBLOCK file support | |
| 18 | |
| 19 The VARBLOCK file format is as follows, where || denotes byte concatenation: | |
| 20 | |
| 21 FILE := VERSION || BLOCK || BLOCK ... | |
| 22 | |
| 23 BLOCK := LENGTH || DATA | |
| 24 | |
| 25 LENGTH := varint-encoded length of the subsequent data. Varint comes from | |
| 26 Google Protobuf, and encodes an integer into a variable number of bytes. | |
| 27 Each byte uses the 7 lowest bits to encode the value. The highest bit set | |
| 28 to 1 indicates the next byte is also part of the varint. The last byte will | |
| 29 have this bit set to 0. | |
| 30 | |
| 31 This file format is called the VARBLOCK format, in line with the varint format | |
| 32 used to denote the block sizes. | |
| 33 | |
| 34 ''' | |
| 35 | |
| 36 from rsa._compat import byte, b | |
| 37 | |
| 38 | |
| 39 ZERO_BYTE = b('\x00') | |
| 40 VARBLOCK_VERSION = 1 | |
| 41 | |
| 42 def read_varint(infile): | |
| 43 '''Reads a varint from the file. | |
| 44 | |
| 45 When the first byte to be read indicates EOF, (0, 0) is returned. When an | |
| 46 EOF occurs when at least one byte has been read, an EOFError exception is | |
| 47 raised. | |
| 48 | |
| 49 @param infile: the file-like object to read from. It should have a read() | |
| 50 method. | |
| 51 @returns (varint, length), the read varint and the number of read bytes. | |
| 52 ''' | |
| 53 | |
| 54 varint = 0 | |
| 55 read_bytes = 0 | |
| 56 | |
| 57 while True: | |
| 58 char = infile.read(1) | |
| 59 if len(char) == 0: | |
| 60 if read_bytes == 0: | |
| 61 return (0, 0) | |
| 62 raise EOFError('EOF while reading varint, value is %i so far' % | |
| 63 varint) | |
| 64 | |
| 65 byte = ord(char) | |
| 66 varint += (byte & 0x7F) << (7 * read_bytes) | |
| 67 | |
| 68 read_bytes += 1 | |
| 69 | |
| 70 if not byte & 0x80: | |
| 71 return (varint, read_bytes) | |
| 72 | |
| 73 | |
| 74 def write_varint(outfile, value): | |
| 75 '''Writes a varint to a file. | |
| 76 | |
| 77 @param outfile: the file-like object to write to. It should have a write() | |
| 78 method. | |
| 79 @returns the number of written bytes. | |
| 80 ''' | |
| 81 | |
| 82 # there is a big difference between 'write the value 0' (this case) and | |
| 83 # 'there is nothing left to write' (the false-case of the while loop) | |
| 84 | |
| 85 if value == 0: | |
| 86 outfile.write(ZERO_BYTE) | |
| 87 return 1 | |
| 88 | |
| 89 written_bytes = 0 | |
| 90 while value > 0: | |
| 91 to_write = value & 0x7f | |
| 92 value = value >> 7 | |
| 93 | |
| 94 if value > 0: | |
| 95 to_write |= 0x80 | |
| 96 | |
| 97 outfile.write(byte(to_write)) | |
| 98 written_bytes += 1 | |
| 99 | |
| 100 return written_bytes | |
| 101 | |
| 102 | |
| 103 def yield_varblocks(infile): | |
| 104 '''Generator, yields each block in the input file. | |
| 105 | |
| 106 @param infile: file to read, is expected to have the VARBLOCK format as | |
| 107 described in the module's docstring. | |
| 108 @yields the contents of each block. | |
| 109 ''' | |
| 110 | |
| 111 # Check the version number | |
| 112 first_char = infile.read(1) | |
| 113 if len(first_char) == 0: | |
| 114 raise EOFError('Unable to read VARBLOCK version number') | |
| 115 | |
| 116 version = ord(first_char) | |
| 117 if version != VARBLOCK_VERSION: | |
| 118 raise ValueError('VARBLOCK version %i not supported' % version) | |
| 119 | |
| 120 while True: | |
| 121 (block_size, read_bytes) = read_varint(infile) | |
| 122 | |
| 123 # EOF at block boundary, that's fine. | |
| 124 if read_bytes == 0 and block_size == 0: | |
| 125 break | |
| 126 | |
| 127 block = infile.read(block_size) | |
| 128 | |
| 129 read_size = len(block) | |
| 130 if read_size != block_size: | |
| 131 raise EOFError('Block size is %i, but could read only %i bytes' % | |
| 132 (block_size, read_size)) | |
| 133 | |
| 134 yield block | |
| 135 | |
| 136 | |
| 137 def yield_fixedblocks(infile, blocksize): | |
| 138 '''Generator, yields each block of ``blocksize`` bytes in the input file. | |
| 139 | |
| 140 :param infile: file to read and separate in blocks. | |
| 141 :returns: a generator that yields the contents of each block | |
| 142 ''' | |
| 143 | |
| 144 while True: | |
| 145 block = infile.read(blocksize) | |
| 146 | |
| 147 read_bytes = len(block) | |
| 148 if read_bytes == 0: | |
| 149 break | |
| 150 | |
| 151 yield block | |
| 152 | |
| 153 if read_bytes < blocksize: | |
| 154 break | |
| 155 | |
| OLD | NEW |