Index: third_party/protobuf/src/google/protobuf/io/coded_stream.cc |
diff --git a/third_party/protobuf/src/google/protobuf/io/coded_stream.cc b/third_party/protobuf/src/google/protobuf/io/coded_stream.cc |
index 93748ee3d8c49ea039b72bb78f569598e50d7cc8..d8354c1f9cef5d46543873e229852bffb178472e 100644 |
--- a/third_party/protobuf/src/google/protobuf/io/coded_stream.cc |
+++ b/third_party/protobuf/src/google/protobuf/io/coded_stream.cc |
@@ -376,49 +376,6 @@ inline ::std::pair<bool, const uint8*> ReadVarint32FromArray( |
return std::make_pair(true, ptr); |
} |
-GOOGLE_ATTRIBUTE_ALWAYS_INLINE::std::pair<bool, const uint8*> ReadVarint64FromArray( |
- const uint8* buffer, uint64* value); |
-inline ::std::pair<bool, const uint8*> ReadVarint64FromArray( |
- const uint8* buffer, uint64* value) { |
- const uint8* ptr = buffer; |
- uint32 b; |
- |
- // Splitting into 32-bit pieces gives better performance on 32-bit |
- // processors. |
- uint32 part0 = 0, part1 = 0, part2 = 0; |
- |
- b = *(ptr++); part0 = b ; if (!(b & 0x80)) goto done; |
- part0 -= 0x80; |
- b = *(ptr++); part0 += b << 7; if (!(b & 0x80)) goto done; |
- part0 -= 0x80 << 7; |
- b = *(ptr++); part0 += b << 14; if (!(b & 0x80)) goto done; |
- part0 -= 0x80 << 14; |
- b = *(ptr++); part0 += b << 21; if (!(b & 0x80)) goto done; |
- part0 -= 0x80 << 21; |
- b = *(ptr++); part1 = b ; if (!(b & 0x80)) goto done; |
- part1 -= 0x80; |
- b = *(ptr++); part1 += b << 7; if (!(b & 0x80)) goto done; |
- part1 -= 0x80 << 7; |
- b = *(ptr++); part1 += b << 14; if (!(b & 0x80)) goto done; |
- part1 -= 0x80 << 14; |
- b = *(ptr++); part1 += b << 21; if (!(b & 0x80)) goto done; |
- part1 -= 0x80 << 21; |
- b = *(ptr++); part2 = b ; if (!(b & 0x80)) goto done; |
- part2 -= 0x80; |
- b = *(ptr++); part2 += b << 7; if (!(b & 0x80)) goto done; |
- // "part2 -= 0x80 << 7" is irrelevant because (0x80 << 7) << 56 is 0. |
- |
- // We have overrun the maximum size of a varint (10 bytes). Assume |
- // the data is corrupt. |
- return std::make_pair(false, ptr); |
- |
- done: |
- *value = (static_cast<uint64>(part0)) | |
- (static_cast<uint64>(part1) << 28) | |
- (static_cast<uint64>(part2) << 56); |
- return std::make_pair(true, ptr); |
-} |
- |
} // namespace |
bool CodedInputStream::ReadVarint32Slow(uint32* value) { |
@@ -451,32 +408,6 @@ int64 CodedInputStream::ReadVarint32Fallback(uint32 first_byte_or_zero) { |
} |
} |
-int CodedInputStream::ReadVarintSizeAsIntSlow() { |
- // Directly invoke ReadVarint64Fallback, since we already tried to optimize |
- // for one-byte varints. |
- std::pair<uint64, bool> p = ReadVarint64Fallback(); |
- if (!p.second || p.first > static_cast<uint64>(INT_MAX)) return -1; |
- return p.first; |
-} |
- |
-int CodedInputStream::ReadVarintSizeAsIntFallback() { |
- if (BufferSize() >= kMaxVarintBytes || |
- // Optimization: We're also safe if the buffer is non-empty and it ends |
- // with a byte that would terminate a varint. |
- (buffer_end_ > buffer_ && !(buffer_end_[-1] & 0x80))) { |
- uint64 temp; |
- ::std::pair<bool, const uint8*> p = ReadVarint64FromArray(buffer_, &temp); |
- if (!p.first || temp > static_cast<uint64>(INT_MAX)) return -1; |
- buffer_ = p.second; |
- return temp; |
- } else { |
- // Really slow case: we will incur the cost of an extra function call here, |
- // but moving this out of line reduces the size of this function, which |
- // improves the common case. In micro benchmarks, this is worth about 10-15% |
- return ReadVarintSizeAsIntSlow(); |
- } |
-} |
- |
uint32 CodedInputStream::ReadTagSlow() { |
if (buffer_ == buffer_end_) { |
// Call refresh. |
@@ -549,15 +480,9 @@ bool CodedInputStream::ReadVarint64Slow(uint64* value) { |
uint32 b; |
do { |
- if (count == kMaxVarintBytes) { |
- *value = 0; |
- return false; |
- } |
+ if (count == kMaxVarintBytes) return false; |
while (buffer_ == buffer_end_) { |
- if (!Refresh()) { |
- *value = 0; |
- return false; |
- } |
+ if (!Refresh()) return false; |
} |
b = *buffer_; |
result |= static_cast<uint64>(b & 0x7F) << (7 * count); |
@@ -574,13 +499,47 @@ std::pair<uint64, bool> CodedInputStream::ReadVarint64Fallback() { |
// Optimization: We're also safe if the buffer is non-empty and it ends |
// with a byte that would terminate a varint. |
(buffer_end_ > buffer_ && !(buffer_end_[-1] & 0x80))) { |
- uint64 temp; |
- ::std::pair<bool, const uint8*> p = ReadVarint64FromArray(buffer_, &temp); |
- if (!p.first) { |
- return std::make_pair(0, false); |
- } |
- buffer_ = p.second; |
- return std::make_pair(temp, true); |
+ // Fast path: We have enough bytes left in the buffer to guarantee that |
+ // this read won't cross the end, so we can skip the checks. |
+ |
+ const uint8* ptr = buffer_; |
+ uint32 b; |
+ |
+ // Splitting into 32-bit pieces gives better performance on 32-bit |
+ // processors. |
+ uint32 part0 = 0, part1 = 0, part2 = 0; |
+ |
+ b = *(ptr++); part0 = b ; if (!(b & 0x80)) goto done; |
+ part0 -= 0x80; |
+ b = *(ptr++); part0 += b << 7; if (!(b & 0x80)) goto done; |
+ part0 -= 0x80 << 7; |
+ b = *(ptr++); part0 += b << 14; if (!(b & 0x80)) goto done; |
+ part0 -= 0x80 << 14; |
+ b = *(ptr++); part0 += b << 21; if (!(b & 0x80)) goto done; |
+ part0 -= 0x80 << 21; |
+ b = *(ptr++); part1 = b ; if (!(b & 0x80)) goto done; |
+ part1 -= 0x80; |
+ b = *(ptr++); part1 += b << 7; if (!(b & 0x80)) goto done; |
+ part1 -= 0x80 << 7; |
+ b = *(ptr++); part1 += b << 14; if (!(b & 0x80)) goto done; |
+ part1 -= 0x80 << 14; |
+ b = *(ptr++); part1 += b << 21; if (!(b & 0x80)) goto done; |
+ part1 -= 0x80 << 21; |
+ b = *(ptr++); part2 = b ; if (!(b & 0x80)) goto done; |
+ part2 -= 0x80; |
+ b = *(ptr++); part2 += b << 7; if (!(b & 0x80)) goto done; |
+ // "part2 -= 0x80 << 7" is irrelevant because (0x80 << 7) << 56 is 0. |
+ |
+ // We have overrun the maximum size of a varint (10 bytes). The data |
+ // must be corrupt. |
+ return std::make_pair(0, false); |
+ |
+ done: |
+ Advance(ptr - buffer_); |
+ return std::make_pair((static_cast<uint64>(part0)) | |
+ (static_cast<uint64>(part1) << 28) | |
+ (static_cast<uint64>(part2) << 56), |
+ true); |
} else { |
uint64 temp; |
bool success = ReadVarint64Slow(&temp); |
@@ -607,13 +566,13 @@ bool CodedInputStream::Refresh() { |
if (total_bytes_warning_threshold_ >= 0 && |
total_bytes_read_ >= total_bytes_warning_threshold_) { |
- GOOGLE_LOG(INFO) << "Reading dangerously large protocol message. If the " |
- "message turns out to be larger than " |
- << total_bytes_limit_ << " bytes, parsing will be halted " |
- "for security reasons. To increase the limit (or to " |
- "disable these warnings), see " |
- "CodedInputStream::SetTotalBytesLimit() in " |
- "google/protobuf/io/coded_stream.h."; |
+ GOOGLE_LOG(WARNING) << "Reading dangerously large protocol message. If the " |
+ "message turns out to be larger than " |
+ << total_bytes_limit_ << " bytes, parsing will be halted " |
+ "for security reasons. To increase the limit (or to " |
+ "disable these warnings), see " |
+ "CodedInputStream::SetTotalBytesLimit() in " |
+ "google/protobuf/io/coded_stream.h."; |
// Don't warn again for this stream, and print total size at the end. |
total_bytes_warning_threshold_ = -2; |
@@ -655,16 +614,13 @@ bool CodedInputStream::Refresh() { |
// CodedOutputStream ================================================= |
-bool CodedOutputStream::default_serialization_deterministic_ = false; |
- |
CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output) |
: output_(output), |
buffer_(NULL), |
buffer_size_(0), |
total_bytes_(0), |
had_error_(false), |
- aliasing_enabled_(false), |
- serialization_deterministic_is_overridden_(false) { |
+ aliasing_enabled_(false) { |
// Eagerly Refresh() so buffer space is immediately available. |
Refresh(); |
// The Refresh() may have failed. If the client doesn't write any data, |
@@ -680,8 +636,7 @@ CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output, |
buffer_size_(0), |
total_bytes_(0), |
had_error_(false), |
- aliasing_enabled_(false), |
- serialization_deterministic_is_overridden_(false) { |
+ aliasing_enabled_(false) { |
if (do_eager_refresh) { |
// Eagerly Refresh() so buffer space is immediately available. |
Refresh(); |
@@ -794,12 +749,104 @@ void CodedOutputStream::WriteVarint32SlowPath(uint32 value) { |
WriteRaw(bytes, size); |
} |
-void CodedOutputStream::WriteVarint64SlowPath(uint64 value) { |
- uint8 bytes[kMaxVarintBytes]; |
- uint8* target = &bytes[0]; |
- uint8* end = WriteVarint64ToArray(value, target); |
- int size = end - target; |
- WriteRaw(bytes, size); |
+inline uint8* CodedOutputStream::WriteVarint64ToArrayInline( |
+ uint64 value, uint8* target) { |
+ // Splitting into 32-bit pieces gives better performance on 32-bit |
+ // processors. |
+ uint32 part0 = static_cast<uint32>(value ); |
+ uint32 part1 = static_cast<uint32>(value >> 28); |
+ uint32 part2 = static_cast<uint32>(value >> 56); |
+ |
+ int size; |
+ |
+ // Here we can't really optimize for small numbers, since the value is |
+ // split into three parts. Cheking for numbers < 128, for instance, |
+ // would require three comparisons, since you'd have to make sure part1 |
+ // and part2 are zero. However, if the caller is using 64-bit integers, |
+ // it is likely that they expect the numbers to often be very large, so |
+ // we probably don't want to optimize for small numbers anyway. Thus, |
+ // we end up with a hardcoded binary search tree... |
+ if (part2 == 0) { |
+ if (part1 == 0) { |
+ if (part0 < (1 << 14)) { |
+ if (part0 < (1 << 7)) { |
+ size = 1; goto size1; |
+ } else { |
+ size = 2; goto size2; |
+ } |
+ } else { |
+ if (part0 < (1 << 21)) { |
+ size = 3; goto size3; |
+ } else { |
+ size = 4; goto size4; |
+ } |
+ } |
+ } else { |
+ if (part1 < (1 << 14)) { |
+ if (part1 < (1 << 7)) { |
+ size = 5; goto size5; |
+ } else { |
+ size = 6; goto size6; |
+ } |
+ } else { |
+ if (part1 < (1 << 21)) { |
+ size = 7; goto size7; |
+ } else { |
+ size = 8; goto size8; |
+ } |
+ } |
+ } |
+ } else { |
+ if (part2 < (1 << 7)) { |
+ size = 9; goto size9; |
+ } else { |
+ size = 10; goto size10; |
+ } |
+ } |
+ |
+ GOOGLE_LOG(FATAL) << "Can't get here."; |
+ |
+ size10: target[9] = static_cast<uint8>((part2 >> 7) | 0x80); |
+ size9 : target[8] = static_cast<uint8>((part2 ) | 0x80); |
+ size8 : target[7] = static_cast<uint8>((part1 >> 21) | 0x80); |
+ size7 : target[6] = static_cast<uint8>((part1 >> 14) | 0x80); |
+ size6 : target[5] = static_cast<uint8>((part1 >> 7) | 0x80); |
+ size5 : target[4] = static_cast<uint8>((part1 ) | 0x80); |
+ size4 : target[3] = static_cast<uint8>((part0 >> 21) | 0x80); |
+ size3 : target[2] = static_cast<uint8>((part0 >> 14) | 0x80); |
+ size2 : target[1] = static_cast<uint8>((part0 >> 7) | 0x80); |
+ size1 : target[0] = static_cast<uint8>((part0 ) | 0x80); |
+ |
+ target[size-1] &= 0x7F; |
+ return target + size; |
+} |
+ |
+void CodedOutputStream::WriteVarint64(uint64 value) { |
+ if (buffer_size_ >= kMaxVarintBytes) { |
+ // Fast path: We have enough bytes left in the buffer to guarantee that |
+ // this write won't cross the end, so we can skip the checks. |
+ uint8* target = buffer_; |
+ |
+ uint8* end = WriteVarint64ToArrayInline(value, target); |
+ int size = end - target; |
+ Advance(size); |
+ } else { |
+ // Slow path: This write might cross the end of the buffer, so we |
+ // compose the bytes first then use WriteRaw(). |
+ uint8 bytes[kMaxVarintBytes]; |
+ int size = 0; |
+ while (value > 0x7F) { |
+ bytes[size++] = (static_cast<uint8>(value) & 0x7F) | 0x80; |
+ value >>= 7; |
+ } |
+ bytes[size++] = static_cast<uint8>(value) & 0x7F; |
+ WriteRaw(bytes, size); |
+ } |
+} |
+ |
+uint8* CodedOutputStream::WriteVarint64ToArray( |
+ uint64 value, uint8* target) { |
+ return WriteVarint64ToArrayInline(value, target); |
} |
bool CodedOutputStream::Refresh() { |
@@ -816,23 +863,46 @@ bool CodedOutputStream::Refresh() { |
} |
} |
-size_t CodedOutputStream::VarintSize32Fallback(uint32 value) { |
- // This computes floor(log2(value)) / 7 + 1 |
- // Use an explicit multiplication to implement the divide of |
- // a number in the 1..31 range. |
- GOOGLE_DCHECK_NE(0, value); // This is enforced by our caller. |
- |
- uint32 log2value = Bits::Log2FloorNonZero(value); |
- return static_cast<size_t>((log2value * 9 + 73) / 64); |
+int CodedOutputStream::VarintSize32Fallback(uint32 value) { |
+ if (value < (1 << 7)) { |
+ return 1; |
+ } else if (value < (1 << 14)) { |
+ return 2; |
+ } else if (value < (1 << 21)) { |
+ return 3; |
+ } else if (value < (1 << 28)) { |
+ return 4; |
+ } else { |
+ return 5; |
+ } |
} |
-size_t CodedOutputStream::VarintSize64(uint64 value) { |
- // This computes value == 0 ? 1 : floor(log2(value)) / 7 + 1 |
- // Use an explicit multiplication to implement the divide of |
- // a number in the 1..63 range. |
- // Explicit OR 0x1 to avoid calling clz(0), which is undefined. |
- uint32 log2value = Bits::Log2FloorNonZero64(value | 0x1); |
- return static_cast<size_t>((log2value * 9 + 73) / 64); |
+int CodedOutputStream::VarintSize64(uint64 value) { |
+ if (value < (1ull << 35)) { |
+ if (value < (1ull << 7)) { |
+ return 1; |
+ } else if (value < (1ull << 14)) { |
+ return 2; |
+ } else if (value < (1ull << 21)) { |
+ return 3; |
+ } else if (value < (1ull << 28)) { |
+ return 4; |
+ } else { |
+ return 5; |
+ } |
+ } else { |
+ if (value < (1ull << 42)) { |
+ return 6; |
+ } else if (value < (1ull << 49)) { |
+ return 7; |
+ } else if (value < (1ull << 56)) { |
+ return 8; |
+ } else if (value < (1ull << 63)) { |
+ return 9; |
+ } else { |
+ return 10; |
+ } |
+ } |
} |
uint8* CodedOutputStream::WriteStringWithSizeToArray(const string& str, |