Index: third_party/protobuf/src/google/protobuf/io/coded_stream.h |
diff --git a/third_party/protobuf/src/google/protobuf/io/coded_stream.h b/third_party/protobuf/src/google/protobuf/io/coded_stream.h |
index c81a33ac6b25c459819fa4c48a0b885675e194cf..dc42e2fe94a1fbff576ee86cf34e1e6b0186811f 100644 |
--- a/third_party/protobuf/src/google/protobuf/io/coded_stream.h |
+++ b/third_party/protobuf/src/google/protobuf/io/coded_stream.h |
@@ -110,6 +110,7 @@ |
#define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ |
#include <assert.h> |
+#include <climits> |
#include <string> |
#include <utility> |
#ifdef _MSC_VER |
@@ -237,13 +238,28 @@ class LIBPROTOBUF_EXPORT CodedInputStream { |
// Read an unsigned integer with Varint encoding. |
bool ReadVarint64(uint64* value); |
+ // Reads a varint off the wire into an "int". This should be used for reading |
+ // sizes off the wire (sizes of strings, submessages, bytes fields, etc). |
+ // |
+ // The value from the wire is interpreted as unsigned. If its value exceeds |
+ // the representable value of an integer on this platform, instead of |
+ // truncating we return false. Truncating (as performed by ReadVarint32() |
+ // above) is an acceptable approach for fields representing an integer, but |
+ // when we are parsing a size from the wire, truncating the value would result |
+ // in us misparsing the payload. |
+ bool ReadVarintSizeAsInt(int* value); |
+ |
// Read a tag. This calls ReadVarint32() and returns the result, or returns |
- // zero (which is not a valid tag) if ReadVarint32() fails. Also, it updates |
- // the last tag value, which can be checked with LastTagWas(). |
+ // zero (which is not a valid tag) if ReadVarint32() fails. Also, ReadTag |
+ // (but not ReadTagNoLastTag) updates the last tag value, which can be checked |
+ // with LastTagWas(). |
+ // |
// Always inline because this is only called in one place per parse loop |
// but it is called for every iteration of said loop, so it should be fast. |
// GCC doesn't want to inline this by default. |
GOOGLE_ATTRIBUTE_ALWAYS_INLINE uint32 ReadTag(); |
+ GOOGLE_ATTRIBUTE_ALWAYS_INLINE uint32 ReadTagNoLastTag(); |
+ |
// This usually a faster alternative to ReadTag() when cutoff is a manifest |
// constant. It does particularly well for cutoff >= 127. The first part |
@@ -255,6 +271,8 @@ class LIBPROTOBUF_EXPORT CodedInputStream { |
// to avoid an extra "is tag == 0?" check here.) |
GOOGLE_ATTRIBUTE_ALWAYS_INLINE std::pair<uint32, bool> ReadTagWithCutoff( |
uint32 cutoff); |
+ GOOGLE_ATTRIBUTE_ALWAYS_INLINE std::pair<uint32, bool> ReadTagWithCutoffNoLastTag( |
+ uint32 cutoff); |
// Usually returns true if calling ReadVarint32() now would produce the given |
// value. Will always return false if ReadVarint32() would not return the |
@@ -282,8 +300,10 @@ class LIBPROTOBUF_EXPORT CodedInputStream { |
// zero, and ConsumedEntireMessage() will return true. |
bool ExpectAtEnd(); |
- // If the last call to ReadTag() or ReadTagWithCutoff() returned the |
- // given value, returns true. Otherwise, returns false; |
+ // If the last call to ReadTag() or ReadTagWithCutoff() returned the given |
+ // value, returns true. Otherwise, returns false. |
+ // ReadTagNoLastTag/ReadTagWithCutoffNoLastTag do not preserve the last |
+ // returned value. |
// |
// This is needed because parsers for some types of embedded messages |
// (with field type TYPE_GROUP) don't actually know that they've reached the |
@@ -594,11 +614,20 @@ class LIBPROTOBUF_EXPORT CodedInputStream { |
// if it fails and the uint32 it read otherwise. The latter has a bool |
// indicating success or failure as part of its return type. |
int64 ReadVarint32Fallback(uint32 first_byte_or_zero); |
+ int ReadVarintSizeAsIntFallback(); |
std::pair<uint64, bool> ReadVarint64Fallback(); |
bool ReadVarint32Slow(uint32* value); |
bool ReadVarint64Slow(uint64* value); |
+ int ReadVarintSizeAsIntSlow(); |
bool ReadLittleEndian32Fallback(uint32* value); |
bool ReadLittleEndian64Fallback(uint64* value); |
+ |
+ template<bool update_last_tag> |
+ GOOGLE_ATTRIBUTE_ALWAYS_INLINE uint32 ReadTagImplementation(); |
+ template<bool update_last_tag> |
+ GOOGLE_ATTRIBUTE_ALWAYS_INLINE |
+ std::pair<uint32, bool> ReadTagWithCutoffImplementation(uint32 cutoff); |
+ |
// Fallback/slow methods for reading tags. These do not update last_tag_, |
// but will set legitimate_message_end_ if we are at the end of the input |
// stream. |
@@ -609,7 +638,7 @@ class LIBPROTOBUF_EXPORT CodedInputStream { |
// Return the size of the buffer. |
int BufferSize() const; |
- static const int kDefaultTotalBytesLimit = 64 << 20; // 64MB |
+ static const int kDefaultTotalBytesLimit = INT_MAX; |
static const int kDefaultTotalBytesWarningThreshold = 32 << 20; // 32MB |
@@ -771,17 +800,17 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { |
uint8* target); |
// Returns the number of bytes needed to encode the given value as a varint. |
- static int VarintSize32(uint32 value); |
+ static size_t VarintSize32(uint32 value); |
// Returns the number of bytes needed to encode the given value as a varint. |
- static int VarintSize64(uint64 value); |
+ static size_t VarintSize64(uint64 value); |
// If negative, 10 bytes. Otheriwse, same as VarintSize32(). |
- static int VarintSize32SignExtended(int32 value); |
+ static size_t VarintSize32SignExtended(int32 value); |
// Compile-time equivalent of VarintSize32(). |
template <uint32 Value> |
struct StaticVarintSize32 { |
- static const int value = |
+ static const size_t value = |
(Value < (1 << 7)) |
? 1 |
: (Value < (1 << 14)) |
@@ -800,6 +829,44 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { |
// created. |
bool HadError() const { return had_error_; } |
+ // Deterministic serialization, if requested, guarantees that for a given |
+ // binary, equal messages will always be serialized to the same bytes. This |
+ // implies: |
+ // . repeated serialization of a message will return the same bytes |
+ // . different processes of the same binary (which may be executing on |
+ // different machines) will serialize equal messages to the same bytes. |
+ // |
+ // Note the deterministic serialization is NOT canonical across languages; it |
+ // is also unstable across different builds with schema changes due to unknown |
+ // fields. Users who need canonical serialization, e.g., persistent storage in |
+ // a canonical form, fingerprinting, etc., should define their own |
+ // canonicalization specification and implement the serializer using |
+ // reflection APIs rather than relying on this API. |
+ // |
+ // If determinisitc serialization is requested, the serializer will |
+ // sort map entries by keys in lexicographical order or numerical order. |
+ // (This is an implementation detail and may subject to change.) |
+ // |
+ // There are two ways to determine whether serialization should be |
+ // deterministic for this CodedOutputStream. If SetSerializationDeterministic |
+ // has not yet been called, then the default comes from the global default, |
+ // which is false, until SetDefaultSerializationDeterministic has been called. |
+ // Otherwise, SetSerializationDeterministic has been called, and the last |
+ // value passed to it is all that matters. |
+ void SetSerializationDeterministic(bool value) { |
+ serialization_deterministic_is_overridden_ = true; |
+ serialization_deterministic_override_ = value; |
+ } |
+ // See above. Also, note that users of this CodedOutputStream may need to |
+ // call IsSerializationDeterministic() to serialize in the intended way. This |
+ // CodedOutputStream cannot enforce a desire for deterministic serialization |
+ // by itself. |
+ bool IsSerializationDeterministic() const { |
+ return serialization_deterministic_is_overridden_ ? |
+ serialization_deterministic_override_ : |
+ default_serialization_deterministic_; |
+ } |
+ |
private: |
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedOutputStream); |
@@ -809,6 +876,10 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { |
int total_bytes_; // Sum of sizes of all buffers seen so far. |
bool had_error_; // Whether an error occurred during output. |
bool aliasing_enabled_; // See EnableAliasing(). |
+ // See SetSerializationDeterministic() regarding these three fields. |
+ bool serialization_deterministic_is_overridden_; |
+ bool serialization_deterministic_override_; |
+ static bool default_serialization_deterministic_; |
// Advance the buffer by a given number of bytes. |
void Advance(int amount); |
@@ -824,18 +895,14 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { |
// If this write might cross the end of the buffer, we compose the bytes first |
// then use WriteRaw(). |
void WriteVarint32SlowPath(uint32 value); |
+ void WriteVarint64SlowPath(uint64 value); |
+ |
+ static size_t VarintSize32Fallback(uint32 value); |
- // Always-inlined versions of WriteVarint* functions so that code can be |
- // reused, while still controlling size. For instance, WriteVarint32ToArray() |
- // should not directly call this: since it is inlined itself, doing so |
- // would greatly increase the size of generated code. Instead, it should call |
- // WriteVarint32FallbackToArray. Meanwhile, WriteVarint32() is already |
- // out-of-line, so it should just invoke this directly to avoid any extra |
- // function call overhead. |
- GOOGLE_ATTRIBUTE_ALWAYS_INLINE static uint8* WriteVarint64ToArrayInline( |
- uint64 value, uint8* target); |
- |
- static int VarintSize32Fallback(uint32 value); |
+ // See above. Other projects may use "friend" to allow them to call this. |
+ static void SetDefaultSerializationDeterministic() { |
+ default_serialization_deterministic_ = true; |
+ } |
}; |
// inline methods ==================================================== |
@@ -868,6 +935,19 @@ inline bool CodedInputStream::ReadVarint64(uint64* value) { |
return p.second; |
} |
+inline bool CodedInputStream::ReadVarintSizeAsInt(int* value) { |
+ if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) { |
+ int v = *buffer_; |
+ if (v < 0x80) { |
+ *value = v; |
+ Advance(1); |
+ return true; |
+ } |
+ } |
+ *value = ReadVarintSizeAsIntFallback(); |
+ return *value >= 0; |
+} |
+ |
// static |
inline const uint8* CodedInputStream::ReadLittleEndian32FromArray( |
const uint8* buffer, |
@@ -934,21 +1014,47 @@ inline bool CodedInputStream::ReadLittleEndian64(uint64* value) { |
} |
inline uint32 CodedInputStream::ReadTag() { |
+ return ReadTagImplementation<true>(); |
+} |
+ |
+inline uint32 CodedInputStream::ReadTagNoLastTag() { |
+ return ReadTagImplementation<false>(); |
+} |
+ |
+template<bool update_last_tag> |
+inline uint32 CodedInputStream::ReadTagImplementation() { |
uint32 v = 0; |
if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) { |
v = *buffer_; |
if (v < 0x80) { |
- last_tag_ = v; |
+ if (update_last_tag) { |
+ last_tag_ = v; |
+ } |
Advance(1); |
return v; |
} |
} |
- last_tag_ = ReadTagFallback(v); |
- return last_tag_; |
+ v = ReadTagFallback(v); |
+ if (update_last_tag) { |
+ last_tag_ = v; |
+ } |
+ return v; |
} |
inline std::pair<uint32, bool> CodedInputStream::ReadTagWithCutoff( |
uint32 cutoff) { |
+ return ReadTagWithCutoffImplementation<true>(cutoff); |
+} |
+ |
+inline std::pair<uint32, bool> CodedInputStream::ReadTagWithCutoffNoLastTag( |
+ uint32 cutoff) { |
+ return ReadTagWithCutoffImplementation<false>(cutoff); |
+} |
+ |
+template<bool update_last_tag> |
+inline std::pair<uint32, bool> |
+CodedInputStream::ReadTagWithCutoffImplementation( |
+ uint32 cutoff) { |
// In performance-sensitive code we can expect cutoff to be a compile-time |
// constant, and things like "cutoff >= kMax1ByteVarint" to be evaluated at |
// compile time. |
@@ -960,7 +1066,10 @@ inline std::pair<uint32, bool> CodedInputStream::ReadTagWithCutoff( |
first_byte_or_zero = buffer_[0]; |
if (static_cast<int8>(buffer_[0]) > 0) { |
const uint32 kMax1ByteVarint = 0x7f; |
- uint32 tag = last_tag_ = buffer_[0]; |
+ uint32 tag = buffer_[0]; |
+ if (update_last_tag) { |
+ last_tag_ = tag; |
+ } |
Advance(1); |
return std::make_pair(tag, cutoff >= kMax1ByteVarint || tag <= cutoff); |
} |
@@ -971,7 +1080,10 @@ inline std::pair<uint32, bool> CodedInputStream::ReadTagWithCutoff( |
GOOGLE_PREDICT_TRUE(buffer_ + 1 < buffer_end_) && |
GOOGLE_PREDICT_TRUE((buffer_[0] & ~buffer_[1]) >= 0x80)) { |
const uint32 kMax2ByteVarint = (0x7f << 7) + 0x7f; |
- uint32 tag = last_tag_ = (1u << 7) * buffer_[1] + (buffer_[0] - 0x80); |
+ uint32 tag = (1u << 7) * buffer_[1] + (buffer_[0] - 0x80); |
+ if (update_last_tag) { |
+ last_tag_ = tag; |
+ } |
Advance(2); |
// It might make sense to test for tag == 0 now, but it is so rare that |
// that we don't bother. A varint-encoded 0 should be one byte unless |
@@ -984,8 +1096,11 @@ inline std::pair<uint32, bool> CodedInputStream::ReadTagWithCutoff( |
} |
} |
// Slow path |
- last_tag_ = ReadTagFallback(first_byte_or_zero); |
- return std::make_pair(last_tag_, static_cast<uint32>(last_tag_ - 1) < cutoff); |
+ const uint32 tag = ReadTagFallback(first_byte_or_zero); |
+ if (update_last_tag) { |
+ last_tag_ = tag; |
+ } |
+ return std::make_pair(tag, static_cast<uint32>(tag - 1) < cutoff); |
} |
inline bool CodedInputStream::LastTagWas(uint32 expected) { |
@@ -1080,21 +1195,24 @@ inline uint8* CodedOutputStream::WriteVarint32ToArray(uint32 value, |
return target + 1; |
} |
-inline void CodedOutputStream::WriteVarint32SignExtended(int32 value) { |
- if (value < 0) { |
- WriteVarint64(static_cast<uint64>(value)); |
- } else { |
- WriteVarint32(static_cast<uint32>(value)); |
+inline uint8* CodedOutputStream::WriteVarint64ToArray(uint64 value, |
+ uint8* target) { |
+ while (value >= 0x80) { |
+ *target = static_cast<uint8>(value | 0x80); |
+ value >>= 7; |
+ ++target; |
} |
+ *target = static_cast<uint8>(value); |
+ return target + 1; |
+} |
+ |
+inline void CodedOutputStream::WriteVarint32SignExtended(int32 value) { |
+ WriteVarint64(static_cast<uint64>(value)); |
} |
inline uint8* CodedOutputStream::WriteVarint32SignExtendedToArray( |
int32 value, uint8* target) { |
- if (value < 0) { |
- return WriteVarint64ToArray(static_cast<uint64>(value), target); |
- } else { |
- return WriteVarint32ToArray(static_cast<uint32>(value), target); |
- } |
+ return WriteVarint64ToArray(static_cast<uint64>(value), target); |
} |
inline uint8* CodedOutputStream::WriteLittleEndian32ToArray(uint32 value, |
@@ -1143,6 +1261,19 @@ inline void CodedOutputStream::WriteVarint32(uint32 value) { |
} |
} |
+inline void CodedOutputStream::WriteVarint64(uint64 value) { |
+ if (buffer_size_ >= 10) { |
+ // Fast path: We have enough bytes left in the buffer to guarantee that |
+ // this write won't cross the end, so we can skip the checks. |
+ uint8* target = buffer_; |
+ uint8* end = WriteVarint64ToArray(value, target); |
+ int size = static_cast<int>(end - target); |
+ Advance(size); |
+ } else { |
+ WriteVarint64SlowPath(value); |
+ } |
+} |
+ |
inline void CodedOutputStream::WriteTag(uint32 value) { |
WriteVarint32(value); |
} |
@@ -1152,7 +1283,7 @@ inline uint8* CodedOutputStream::WriteTagToArray( |
return WriteVarint32ToArray(value, target); |
} |
-inline int CodedOutputStream::VarintSize32(uint32 value) { |
+inline size_t CodedOutputStream::VarintSize32(uint32 value) { |
if (value < (1 << 7)) { |
return 1; |
} else { |
@@ -1160,7 +1291,7 @@ inline int CodedOutputStream::VarintSize32(uint32 value) { |
} |
} |
-inline int CodedOutputStream::VarintSize32SignExtended(int32 value) { |
+inline size_t CodedOutputStream::VarintSize32SignExtended(int32 value) { |
if (value < 0) { |
return 10; // TODO(kenton): Make this a symbolic constant. |
} else { |