third_party/protobuf: Update to HEAD (83d681ee2c)

third_party/protobuf/src/google/protobuf/io/coded_stream.h

 // Protocol Buffers - Google's data interchange format
 // Copyright 2008 Google Inc.  All rights reserved.
 // https://developers.google.com/protocol-buffers/
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
@@ skipping 92 matching lines @@
 // bytes as "10101011 00101100".
 //
 // In theory, varint could be used to encode integers of any length.
 // However, for practicality we set a limit at 64 bits.  The maximum encoded
 // length of a number is thus 10 bytes.
 
 #ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_H__
 #define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__
 
 #include <assert.h>
+#include <climits>
 #include <string>
 #include <utility>
 #ifdef _MSC_VER
   // Assuming windows is always little-endian.
   #if !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST)
     #define PROTOBUF_LITTLE_ENDIAN 1
   #endif
   #if _MSC_VER >= 1300 && !defined(__INTEL_COMPILER)
     // If MSVC has "/RTCc" set, it will complain about truncating casts at
     // runtime.  This file contains some intentional truncating casts.
@@ skipping 107 matching lines @@
   static const uint8* ReadLittleEndian64FromArray(const uint8* buffer,
                                                    uint64* value);
 
   // Read an unsigned integer with Varint encoding, truncating to 32 bits.
   // Reading a 32-bit value is equivalent to reading a 64-bit one and casting
   // it to uint32, but may be more efficient.
   bool ReadVarint32(uint32* value);
   // Read an unsigned integer with Varint encoding.
   bool ReadVarint64(uint64* value);
 
+  // Reads a varint off the wire into an "int". This should be used for reading
+  // sizes off the wire (sizes of strings, submessages, bytes fields, etc).
+  //
+  // The value from the wire is interpreted as unsigned.  If its value exceeds
+  // the representable value of an integer on this platform, instead of
+  // truncating we return false. Truncating (as performed by ReadVarint32()
+  // above) is an acceptable approach for fields representing an integer, but
+  // when we are parsing a size from the wire, truncating the value would result
+  // in us misparsing the payload.
+  bool ReadVarintSizeAsInt(int* value);
+
   // Read a tag.  This calls ReadVarint32() and returns the result, or returns
-  // zero (which is not a valid tag) if ReadVarint32() fails.  Also, it updates
-  // the last tag value, which can be checked with LastTagWas().
+  // zero (which is not a valid tag) if ReadVarint32() fails.  Also, ReadTag
+  // (but not ReadTagNoLastTag) updates the last tag value, which can be checked
+  // with LastTagWas().
+  //
   // Always inline because this is only called in one place per parse loop
   // but it is called for every iteration of said loop, so it should be fast.
   // GCC doesn't want to inline this by default.
   GOOGLE_ATTRIBUTE_ALWAYS_INLINE uint32 ReadTag();
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE uint32 ReadTagNoLastTag();
+
 
   // This usually a faster alternative to ReadTag() when cutoff is a manifest
   // constant.  It does particularly well for cutoff >= 127.  The first part
   // of the return value is the tag that was read, though it can also be 0 in
   // the cases where ReadTag() would return 0.  If the second part is true
   // then the tag is known to be in [0, cutoff].  If not, the tag either is
   // above cutoff or is 0.  (There's intentional wiggle room when tag is 0,
   // because that can arise in several ways, and for best performance we want
   // to avoid an extra "is tag == 0?" check here.)
   GOOGLE_ATTRIBUTE_ALWAYS_INLINE std::pair<uint32, bool> ReadTagWithCutoff(
       uint32 cutoff);
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE std::pair<uint32, bool> ReadTagWithCutoffNoLastTag(
+      uint32 cutoff);
 
   // Usually returns true if calling ReadVarint32() now would produce the given
   // value.  Will always return false if ReadVarint32() would not return the
   // given value.  If ExpectTag() returns true, it also advances past
   // the varint.  For best performance, use a compile-time constant as the
   // parameter.
   // Always inline because this collapses to a small number of instructions
   // when given a constant parameter, but GCC doesn't want to inline by default.
   GOOGLE_ATTRIBUTE_ALWAYS_INLINE bool ExpectTag(uint32 expected);
 
   // Like above, except this reads from the specified buffer. The caller is
   // responsible for ensuring that the buffer is large enough to read a varint
   // of the expected size. For best performance, use a compile-time constant as
   // the expected tag parameter.
   //
   // Returns a pointer beyond the expected tag if it was found, or NULL if it
   // was not.
   GOOGLE_ATTRIBUTE_ALWAYS_INLINE static const uint8* ExpectTagFromArray(
       const uint8* buffer,
       uint32 expected);
 
   // Usually returns true if no more bytes can be read.  Always returns false
   // if more bytes can be read.  If ExpectAtEnd() returns true, a subsequent
   // call to LastTagWas() will act as if ReadTag() had been called and returned
   // zero, and ConsumedEntireMessage() will return true.
   bool ExpectAtEnd();
 
-  // If the last call to ReadTag() or ReadTagWithCutoff() returned the
-  // given value, returns true.  Otherwise, returns false;
+  // If the last call to ReadTag() or ReadTagWithCutoff() returned the given
+  // value, returns true.  Otherwise, returns false.
+  // ReadTagNoLastTag/ReadTagWithCutoffNoLastTag do not preserve the last
+  // returned value.
   //
   // This is needed because parsers for some types of embedded messages
   // (with field type TYPE_GROUP) don't actually know that they've reached the
   // end of a message until they see an ENDGROUP tag, which was actually part
   // of the enclosing message.  The enclosing message would like to check that
   // tag to make sure it had the right number, so it calls LastTagWas() on
   // return from the embedded parser to check.
   bool LastTagWas(uint32 expected);
 
   // When parsing message (but NOT a group), this method must be called
@@ skipping 290 matching lines @@
   // piece. The Fallback method is used when we can't use the one-byte
   // optimization. The Slow method is yet another fallback when the buffer is
   // not large enough. Making the slow path out-of-line speeds up the common
   // case by 10-15%. The slow path is fairly uncommon: it only triggers when a
   // message crosses multiple buffers.  Note: ReadVarint32Fallback() and
   // ReadVarint64Fallback() are called frequently and generally not inlined, so
   // they have been optimized to avoid "out" parameters.  The former returns -1
   // if it fails and the uint32 it read otherwise.  The latter has a bool
   // indicating success or failure as part of its return type.
   int64 ReadVarint32Fallback(uint32 first_byte_or_zero);
+  int ReadVarintSizeAsIntFallback();
   std::pair<uint64, bool> ReadVarint64Fallback();
   bool ReadVarint32Slow(uint32* value);
   bool ReadVarint64Slow(uint64* value);
+  int ReadVarintSizeAsIntSlow();
   bool ReadLittleEndian32Fallback(uint32* value);
   bool ReadLittleEndian64Fallback(uint64* value);
+
+  template<bool update_last_tag>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE uint32 ReadTagImplementation();
+  template<bool update_last_tag>
+  GOOGLE_ATTRIBUTE_ALWAYS_INLINE
+  std::pair<uint32, bool> ReadTagWithCutoffImplementation(uint32 cutoff);
+
   // Fallback/slow methods for reading tags. These do not update last_tag_,
   // but will set legitimate_message_end_ if we are at the end of the input
   // stream.
   uint32 ReadTagFallback(uint32 first_byte_or_zero);
   uint32 ReadTagSlow();
   bool ReadStringFallback(string* buffer, int size);
 
   // Return the size of the buffer.
   int BufferSize() const;
 
-  static const int kDefaultTotalBytesLimit = 64 << 20;  // 64MB
+  static const int kDefaultTotalBytesLimit = INT_MAX;
 
   static const int kDefaultTotalBytesWarningThreshold = 32 << 20;  // 32MB
 
   static int default_recursion_limit_;  // 100 by default.
 };
 
 // Class which encodes and writes binary data which is composed of varint-
 // encoded integers and fixed-width pieces.  Wraps a ZeroCopyOutputStream.
 // Most users will not need to deal with CodedOutputStream.
 //
@@ skipping 141 matching lines @@
   // In particular, if the input is a compile-time constant, this method
   // compiles down to a couple instructions.
   // Always inline because otherwise the aformentioned optimization can't work,
   // but GCC by default doesn't want to inline this.
   void WriteTag(uint32 value);
   // Like WriteTag()  but writing directly to the target array.
   GOOGLE_ATTRIBUTE_ALWAYS_INLINE static uint8* WriteTagToArray(uint32 value,
                                                         uint8* target);
 
   // Returns the number of bytes needed to encode the given value as a varint.
-  static int VarintSize32(uint32 value);
+  static size_t VarintSize32(uint32 value);
   // Returns the number of bytes needed to encode the given value as a varint.
-  static int VarintSize64(uint64 value);
+  static size_t VarintSize64(uint64 value);
 
   // If negative, 10 bytes.  Otheriwse, same as VarintSize32().
-  static int VarintSize32SignExtended(int32 value);
+  static size_t VarintSize32SignExtended(int32 value);
 
   // Compile-time equivalent of VarintSize32().
   template <uint32 Value>
   struct StaticVarintSize32 {
-    static const int value =
+    static const size_t value =
         (Value < (1 << 7))
             ? 1
             : (Value < (1 << 14))
                 ? 2
                 : (Value < (1 << 21))
                     ? 3
                     : (Value < (1 << 28))
                         ? 4
                         : 5;
   };
 
   // Returns the total number of bytes written since this object was created.
   inline int ByteCount() const;
 
   // Returns true if there was an underlying I/O error since this object was
   // created.
   bool HadError() const { return had_error_; }
 
+  // Deterministic serialization, if requested, guarantees that for a given
+  // binary, equal messages will always be serialized to the same bytes. This
+  // implies:
+  //   . repeated serialization of a message will return the same bytes
+  //   . different processes of the same binary (which may be executing on
+  //     different machines) will serialize equal messages to the same bytes.
+  //
+  // Note the deterministic serialization is NOT canonical across languages; it
+  // is also unstable across different builds with schema changes due to unknown
+  // fields. Users who need canonical serialization, e.g., persistent storage in
+  // a canonical form, fingerprinting, etc., should define their own
+  // canonicalization specification and implement the serializer using
+  // reflection APIs rather than relying on this API.
+  //
+  // If determinisitc serialization is requested, the serializer will
+  // sort map entries by keys in lexicographical order or numerical order.
+  // (This is an implementation detail and may subject to change.)
+  //
+  // There are two ways to determine whether serialization should be
+  // deterministic for this CodedOutputStream.  If SetSerializationDeterministic
+  // has not yet been called, then the default comes from the global default,
+  // which is false, until SetDefaultSerializationDeterministic has been called.
+  // Otherwise, SetSerializationDeterministic has been called, and the last
+  // value passed to it is all that matters.
+  void SetSerializationDeterministic(bool value) {
+    serialization_deterministic_is_overridden_ = true;
+    serialization_deterministic_override_ = value;
+  }
+  // See above.  Also, note that users of this CodedOutputStream may need to
+  // call IsSerializationDeterministic() to serialize in the intended way.  This
+  // CodedOutputStream cannot enforce a desire for deterministic serialization
+  // by itself.
+  bool IsSerializationDeterministic() const {
+    return serialization_deterministic_is_overridden_ ?
+        serialization_deterministic_override_ :
+        default_serialization_deterministic_;
+  }
+
  private:
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedOutputStream);
 
   ZeroCopyOutputStream* output_;
   uint8* buffer_;
   int buffer_size_;
   int total_bytes_;  // Sum of sizes of all buffers seen so far.
   bool had_error_;   // Whether an error occurred during output.
   bool aliasing_enabled_;  // See EnableAliasing().
+  // See SetSerializationDeterministic() regarding these three fields.
+  bool serialization_deterministic_is_overridden_;
+  bool serialization_deterministic_override_;
+  static bool default_serialization_deterministic_;
 
   // Advance the buffer by a given number of bytes.
   void Advance(int amount);
 
   // Called when the buffer runs out to request more data.  Implies an
   // Advance(buffer_size_).
   bool Refresh();
 
   // Like WriteRaw() but may avoid copying if the underlying
   // ZeroCopyOutputStream supports it.
   void WriteAliasedRaw(const void* buffer, int size);
 
   // If this write might cross the end of the buffer, we compose the bytes first
   // then use WriteRaw().
   void WriteVarint32SlowPath(uint32 value);
+  void WriteVarint64SlowPath(uint64 value);
 
-  // Always-inlined versions of WriteVarint* functions so that code can be
-  // reused, while still controlling size. For instance, WriteVarint32ToArray()
-  // should not directly call this: since it is inlined itself, doing so
-  // would greatly increase the size of generated code. Instead, it should call
-  // WriteVarint32FallbackToArray.  Meanwhile, WriteVarint32() is already
-  // out-of-line, so it should just invoke this directly to avoid any extra
-  // function call overhead.
-  GOOGLE_ATTRIBUTE_ALWAYS_INLINE static uint8* WriteVarint64ToArrayInline(
-      uint64 value, uint8* target);
+  static size_t VarintSize32Fallback(uint32 value);
 
-  static int VarintSize32Fallback(uint32 value);
+  // See above.  Other projects may use "friend" to allow them to call this.
+  static void SetDefaultSerializationDeterministic() {
+    default_serialization_deterministic_ = true;
+  }
 };
 
 // inline methods ====================================================
 // The vast majority of varints are only one byte.  These inline
 // methods optimize for that case.
 
 inline bool CodedInputStream::ReadVarint32(uint32* value) {
   uint32 v = 0;
   if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) {
     v = *buffer_;
@@ skipping 12 matching lines @@
   if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_) && *buffer_ < 0x80) {
     *value = *buffer_;
     Advance(1);
     return true;
   }
   std::pair<uint64, bool> p = ReadVarint64Fallback();
   *value = p.first;
   return p.second;
 }
 
+inline bool CodedInputStream::ReadVarintSizeAsInt(int* value) {
+  if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) {
+    int v = *buffer_;
+    if (v < 0x80) {
+      *value = v;
+      Advance(1);
+      return true;
+    }
+  }
+  *value = ReadVarintSizeAsIntFallback();
+  return *value >= 0;
+}
+
 // static
 inline const uint8* CodedInputStream::ReadLittleEndian32FromArray(
     const uint8* buffer,
     uint32* value) {
 #if defined(PROTOBUF_LITTLE_ENDIAN)
   memcpy(value, buffer, sizeof(*value));
   return buffer + sizeof(*value);
 #else
   *value = (static_cast<uint32>(buffer[0])      ) |
            (static_cast<uint32>(buffer[1]) <<  8) |
@@ skipping 46 matching lines @@
     return true;
   } else {
     return ReadLittleEndian64Fallback(value);
   }
 #else
   return ReadLittleEndian64Fallback(value);
 #endif
 }
 
 inline uint32 CodedInputStream::ReadTag() {
+  return ReadTagImplementation<true>();
+}
+
+inline uint32 CodedInputStream::ReadTagNoLastTag() {
+  return ReadTagImplementation<false>();
+}
+
+template<bool update_last_tag>
+inline uint32 CodedInputStream::ReadTagImplementation() {
   uint32 v = 0;
   if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) {
     v = *buffer_;
     if (v < 0x80) {
-      last_tag_ = v;
+      if (update_last_tag) {
+        last_tag_ = v;
+      }
       Advance(1);
       return v;
     }
   }
-  last_tag_ = ReadTagFallback(v);
-  return last_tag_;
+  v = ReadTagFallback(v);
+  if (update_last_tag) {
+    last_tag_ = v;
+  }
+  return v;
 }
 
 inline std::pair<uint32, bool> CodedInputStream::ReadTagWithCutoff(
     uint32 cutoff) {
+  return ReadTagWithCutoffImplementation<true>(cutoff);
+}
+
+inline std::pair<uint32, bool> CodedInputStream::ReadTagWithCutoffNoLastTag(
+    uint32 cutoff) {
+  return ReadTagWithCutoffImplementation<false>(cutoff);
+}
+
+template<bool update_last_tag>
+inline std::pair<uint32, bool>
+CodedInputStream::ReadTagWithCutoffImplementation(
+    uint32 cutoff) {
   // In performance-sensitive code we can expect cutoff to be a compile-time
   // constant, and things like "cutoff >= kMax1ByteVarint" to be evaluated at
   // compile time.
   uint32 first_byte_or_zero = 0;
   if (GOOGLE_PREDICT_TRUE(buffer_ < buffer_end_)) {
     // Hot case: buffer_ non_empty, buffer_[0] in [1, 128).
     // TODO(gpike): Is it worth rearranging this? E.g., if the number of fields
     // is large enough then is it better to check for the two-byte case first?
     first_byte_or_zero = buffer_[0];
     if (static_cast<int8>(buffer_[0]) > 0) {
       const uint32 kMax1ByteVarint = 0x7f;
-      uint32 tag = last_tag_ = buffer_[0];
+      uint32 tag = buffer_[0];
+      if (update_last_tag) {
+        last_tag_ = tag;
+      }
       Advance(1);
       return std::make_pair(tag, cutoff >= kMax1ByteVarint || tag <= cutoff);
     }
     // Other hot case: cutoff >= 0x80, buffer_ has at least two bytes available,
     // and tag is two bytes.  The latter is tested by bitwise-and-not of the
     // first byte and the second byte.
     if (cutoff >= 0x80 &&
         GOOGLE_PREDICT_TRUE(buffer_ + 1 < buffer_end_) &&
         GOOGLE_PREDICT_TRUE((buffer_[0] & ~buffer_[1]) >= 0x80)) {
       const uint32 kMax2ByteVarint = (0x7f << 7) + 0x7f;
-      uint32 tag = last_tag_ = (1u << 7) * buffer_[1] + (buffer_[0] - 0x80);
+      uint32 tag = (1u << 7) * buffer_[1] + (buffer_[0] - 0x80);
+      if (update_last_tag) {
+        last_tag_ = tag;
+      }
       Advance(2);
       // It might make sense to test for tag == 0 now, but it is so rare that
       // that we don't bother.  A varint-encoded 0 should be one byte unless
       // the encoder lost its mind.  The second part of the return value of
       // this function is allowed to be either true or false if the tag is 0,
       // so we don't have to check for tag == 0.  We may need to check whether
       // it exceeds cutoff.
       bool at_or_below_cutoff = cutoff >= kMax2ByteVarint || tag <= cutoff;
       return std::make_pair(tag, at_or_below_cutoff);
     }
   }
   // Slow path
-  last_tag_ = ReadTagFallback(first_byte_or_zero);
-  return std::make_pair(last_tag_, static_cast<uint32>(last_tag_ - 1) < cutoff);
+  const uint32 tag = ReadTagFallback(first_byte_or_zero);
+  if (update_last_tag) {
+    last_tag_ = tag;
+  }
+  return std::make_pair(tag, static_cast<uint32>(tag - 1) < cutoff);
 }
 
 inline bool CodedInputStream::LastTagWas(uint32 expected) {
   return last_tag_ == expected;
 }
 
 inline bool CodedInputStream::ConsumedEntireMessage() {
   return legitimate_message_end_;
 }
 
@@ skipping 74 matching lines @@
                                                       uint8* target) {
   while (value >= 0x80) {
     *target = static_cast<uint8>(value | 0x80);
     value >>= 7;
     ++target;
   }
   *target = static_cast<uint8>(value);
   return target + 1;
 }
 
+inline uint8* CodedOutputStream::WriteVarint64ToArray(uint64 value,
+                                                      uint8* target) {
+  while (value >= 0x80) {
+    *target = static_cast<uint8>(value | 0x80);
+    value >>= 7;
+    ++target;
+  }
+  *target = static_cast<uint8>(value);
+  return target + 1;
+}
+
 inline void CodedOutputStream::WriteVarint32SignExtended(int32 value) {
-  if (value < 0) {
-    WriteVarint64(static_cast<uint64>(value));
-  } else {
-    WriteVarint32(static_cast<uint32>(value));
-  }
+  WriteVarint64(static_cast<uint64>(value));
 }
 
 inline uint8* CodedOutputStream::WriteVarint32SignExtendedToArray(
     int32 value, uint8* target) {
-  if (value < 0) {
-    return WriteVarint64ToArray(static_cast<uint64>(value), target);
-  } else {
-    return WriteVarint32ToArray(static_cast<uint32>(value), target);
-  }
+  return WriteVarint64ToArray(static_cast<uint64>(value), target);
 }
 
 inline uint8* CodedOutputStream::WriteLittleEndian32ToArray(uint32 value,
                                                             uint8* target) {
 #if defined(PROTOBUF_LITTLE_ENDIAN)
   memcpy(target, &value, sizeof(value));
 #else
   target[0] = static_cast<uint8>(value);
   target[1] = static_cast<uint8>(value >>  8);
   target[2] = static_cast<uint8>(value >> 16);
@@ skipping 28 matching lines @@
     // this write won't cross the end, so we can skip the checks.
     uint8* target = buffer_;
     uint8* end = WriteVarint32ToArray(value, target);
     int size = static_cast<int>(end - target);
     Advance(size);
   } else {
     WriteVarint32SlowPath(value);
   }
 }
 
+inline void CodedOutputStream::WriteVarint64(uint64 value) {
+  if (buffer_size_ >= 10) {
+    // Fast path:  We have enough bytes left in the buffer to guarantee that
+    // this write won't cross the end, so we can skip the checks.
+    uint8* target = buffer_;
+    uint8* end = WriteVarint64ToArray(value, target);
+    int size = static_cast<int>(end - target);
+    Advance(size);
+  } else {
+    WriteVarint64SlowPath(value);
+  }
+}
+
 inline void CodedOutputStream::WriteTag(uint32 value) {
   WriteVarint32(value);
 }
 
 inline uint8* CodedOutputStream::WriteTagToArray(
     uint32 value, uint8* target) {
   return WriteVarint32ToArray(value, target);
 }
 
-inline int CodedOutputStream::VarintSize32(uint32 value) {
+inline size_t CodedOutputStream::VarintSize32(uint32 value) {
   if (value < (1 << 7)) {
     return 1;
   } else  {
     return VarintSize32Fallback(value);
   }
 }
 
-inline int CodedOutputStream::VarintSize32SignExtended(int32 value) {
+inline size_t CodedOutputStream::VarintSize32SignExtended(int32 value) {
   if (value < 0) {
     return 10;     // TODO(kenton):  Make this a symbolic constant.
   } else {
     return VarintSize32(static_cast<uint32>(value));
   }
 }
 
 inline void CodedOutputStream::WriteString(const string& str) {
   WriteRaw(str.data(), static_cast<int>(str.size()));
 }
@@ skipping 111 matching lines @@
 }  // namespace io
 }  // namespace protobuf
 
 
 #if _MSC_VER >= 1300 && !defined(__INTEL_COMPILER)
   #pragma runtime_checks("c", restore)
 #endif  // _MSC_VER && !defined(__INTEL_COMPILER)
 
 }  // namespace google
 #endif  // GOOGLE_PROTOBUF_IO_CODED_STREAM_H__