Update //third_party/protobuf to version 3.

third_party/protobuf/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs

Index: third_party/protobuf/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs
diff --git a/third_party/protobuf/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs b/third_party/protobuf/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs
new file mode 100644
index 0000000000000000000000000000000000000000..a0a62227684904418945b4bd404c92458293bc15
--- /dev/null
+++ b/third_party/protobuf/csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs
@@ -0,0 +1,409 @@
+#region Copyright notice and license
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#endregion
+using NUnit.Framework;
+using System;
+using System.IO;
+
+namespace Google.Protobuf
+{
+    public class JsonTokenizerTest
+    {
+        [Test]
+        public void EmptyObjectValue()
+        {
+            AssertTokens("{}", JsonToken.StartObject, JsonToken.EndObject);
+        }
+
+        [Test]
+        public void EmptyArrayValue()
+        {
+            AssertTokens("[]", JsonToken.StartArray, JsonToken.EndArray);
+        }
+
+        [Test]
+        [TestCase("foo", "foo")]
+        [TestCase("tab\\t", "tab\t")]
+        [TestCase("line\\nfeed", "line\nfeed")]
+        [TestCase("carriage\\rreturn", "carriage\rreturn")]
+        [TestCase("back\\bspace", "back\bspace")]
+        [TestCase("form\\ffeed", "form\ffeed")]
+        [TestCase("escaped\\/slash", "escaped/slash")]
+        [TestCase("escaped\\\\backslash", "escaped\\backslash")]
+        [TestCase("escaped\\\"quote", "escaped\"quote")]
+        [TestCase("foo {}[] bar", "foo {}[] bar")]
+        [TestCase("foo\\u09aFbar", "foo\u09afbar")] // Digits, upper hex, lower hex
+        [TestCase("ab\ud800\udc00cd", "ab\ud800\udc00cd")]
+        [TestCase("ab\\ud800\\udc00cd", "ab\ud800\udc00cd")]
+        public void StringValue(string json, string expectedValue)
+        {
+            AssertTokensNoReplacement("\"" + json + "\"", JsonToken.Value(expectedValue));
+        }
+
+        // Valid surrogate pairs, with mixed escaping. These test cases can't be expressed
+        // using TestCase as they have no valid UTF-8 representation.
+        // It's unclear exactly how we should handle a mixture of escaped or not: that can't
+        // come from UTF-8 text, but could come from a .NET string. For the moment,
+        // treat it as valid in the obvious way.
+        [Test]
+        public void MixedSurrogatePairs()
+        {
+            string expected = "\ud800\udc00";
+            AssertTokens("'\\ud800\udc00'", JsonToken.Value(expected));
+            AssertTokens("'\ud800\\udc00'", JsonToken.Value(expected));
+        }
+
+        [Test]
+        public void ObjectDepth()
+        {
+            string json = "{ \"foo\": { \"x\": 1, \"y\": [ 0 ] } }";
+            var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json));
+            // If we had more tests like this, I'd introduce a helper method... but for one test, it's not worth it.
+            Assert.AreEqual(0, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.StartObject, tokenizer.Next());
+            Assert.AreEqual(1, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next());
+            Assert.AreEqual(1, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.StartObject, tokenizer.Next());
+            Assert.AreEqual(2, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next());
+            Assert.AreEqual(2, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.Value(1), tokenizer.Next());
+            Assert.AreEqual(2, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.Name("y"), tokenizer.Next());
+            Assert.AreEqual(2, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.StartArray, tokenizer.Next());
+            Assert.AreEqual(2, tokenizer.ObjectDepth); // Depth hasn't changed in array
+            Assert.AreEqual(JsonToken.Value(0), tokenizer.Next());
+            Assert.AreEqual(2, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.EndArray, tokenizer.Next());
+            Assert.AreEqual(2, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.EndObject, tokenizer.Next());
+            Assert.AreEqual(1, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.EndObject, tokenizer.Next());
+            Assert.AreEqual(0, tokenizer.ObjectDepth);
+            Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next());
+            Assert.AreEqual(0, tokenizer.ObjectDepth);
+        }
+
+        [Test]
+        public void ObjectDepth_WithPushBack()
+        {
+            string json = "{}";
+            var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json));
+            Assert.AreEqual(0, tokenizer.ObjectDepth);
+            var token = tokenizer.Next();
+            Assert.AreEqual(1, tokenizer.ObjectDepth);
+            // When we push back a "start object", we should effectively be back to the previous depth.
+            tokenizer.PushBack(token);
+            Assert.AreEqual(0, tokenizer.ObjectDepth);
+            // Read the same token again, and get back to depth 1
+            token = tokenizer.Next();
+            Assert.AreEqual(1, tokenizer.ObjectDepth);
+
+            // Now the same in reverse, with EndObject
+            token = tokenizer.Next();
+            Assert.AreEqual(0, tokenizer.ObjectDepth);
+            tokenizer.PushBack(token);
+            Assert.AreEqual(1, tokenizer.ObjectDepth);
+            tokenizer.Next();
+            Assert.AreEqual(0, tokenizer.ObjectDepth);
+        }
+
+        [Test]
+        [TestCase("embedded tab\t")]
+        [TestCase("embedded CR\r")]
+        [TestCase("embedded LF\n")]
+        [TestCase("embedded bell\u0007")]
+        [TestCase("bad escape\\a")]
+        [TestCase("incomplete escape\\")]
+        [TestCase("incomplete Unicode escape\\u000")]
+        [TestCase("invalid Unicode escape\\u000H")]
+        // Surrogate pair handling, both in raw .NET strings and escaped. We only need
+        // to detect this in strings, as non-ASCII characters anywhere other than in strings
+        // will already lead to parsing errors.
+        [TestCase("\\ud800")]
+        [TestCase("\\udc00")]
+        [TestCase("\\ud800x")]
+        [TestCase("\\udc00x")]
+        [TestCase("\\udc00\\ud800y")]
+        public void InvalidStringValue(string json)
+        {
+            AssertThrowsAfter("\"" + json + "\"");
+        }
+
+        // Tests for invalid strings that can't be expressed in attributes,
+        // as the constants can't be expressed as UTF-8 strings.
+        [Test]
+        public void InvalidSurrogatePairs()
+        {
+            AssertThrowsAfter("\"\ud800x\"");
+            AssertThrowsAfter("\"\udc00y\"");
+            AssertThrowsAfter("\"\udc00\ud800y\"");
+        }
+
+        [Test]
+        [TestCase("0", 0)]
+        [TestCase("-0", 0)] // We don't distinguish between positive and negative 0
+        [TestCase("1", 1)]
+        [TestCase("-1", -1)]
+        // From here on, assume leading sign is okay...
+        [TestCase("1.125", 1.125)]
+        [TestCase("1.0", 1)]
+        [TestCase("1e5", 100000)]
+        [TestCase("1e000000", 1)] // Weird, but not prohibited by the spec
+        [TestCase("1E5", 100000)]
+        [TestCase("1e+5", 100000)]
+        [TestCase("1E-5", 0.00001)]
+        [TestCase("123E-2", 1.23)]
+        [TestCase("123.45E3", 123450)]
+        [TestCase("   1   ", 1)]
+        public void NumberValue(string json, double expectedValue)
+        {
+            AssertTokens(json, JsonToken.Value(expectedValue));
+        }
+
+        [Test]
+        [TestCase("00")]
+        [TestCase(".5")]
+        [TestCase("1.")]
+        [TestCase("1e")]
+        [TestCase("1e-")]
+        [TestCase("--")]
+        [TestCase("--1")]
+        [TestCase("-1.7977e308")]
+        [TestCase("1.7977e308")]
+        public void InvalidNumberValue(string json)
+        {
+            AssertThrowsAfter(json);
+        }
+
+        [Test]
+        [TestCase("nul")]
+        [TestCase("nothing")]
+        [TestCase("truth")]
+        [TestCase("fALSEhood")]
+        public void InvalidLiterals(string json)
+        {
+            AssertThrowsAfter(json);
+        }
+
+        [Test]
+        public void NullValue()
+        {
+            AssertTokens("null", JsonToken.Null);
+        }
+
+        [Test]
+        public void TrueValue()
+        {
+            AssertTokens("true", JsonToken.True);
+        }
+
+        [Test]
+        public void FalseValue()
+        {
+            AssertTokens("false", JsonToken.False);
+        }
+
+        [Test]
+        public void SimpleObject()
+        {
+            AssertTokens("{'x': 'y'}",
+                JsonToken.StartObject, JsonToken.Name("x"), JsonToken.Value("y"), JsonToken.EndObject);
+        }
+        
+        [Test]
+        [TestCase("[10, 20", 3)]
+        [TestCase("[10,", 2)]
+        [TestCase("[10:20]", 2)]
+        [TestCase("[", 1)]
+        [TestCase("[,", 1)]
+        [TestCase("{", 1)]
+        [TestCase("{,", 1)]
+        [TestCase("{", 1)]
+        [TestCase("{[", 1)]
+        [TestCase("{{", 1)]
+        [TestCase("{0", 1)]
+        [TestCase("{null", 1)]
+        [TestCase("{false", 1)]
+        [TestCase("{true", 1)]
+        [TestCase("}", 0)]
+        [TestCase("]", 0)]
+        [TestCase(",", 0)]
+        [TestCase("'foo' 'bar'", 1)]
+        [TestCase(":", 0)]
+        [TestCase("'foo", 0)] // Incomplete string
+        [TestCase("{ 'foo' }", 2)]
+        [TestCase("{ x:1", 1)] // Property names must be quoted
+        [TestCase("{]", 1)]
+        [TestCase("[}", 1)]
+        [TestCase("[1,", 2)]
+        [TestCase("{'x':0]", 3)]
+        [TestCase("{ 'foo': }", 2)]
+        [TestCase("{ 'foo':'bar', }", 3)]
+        public void InvalidStructure(string json, int expectedValidTokens)
+        {
+            // Note: we don't test that the earlier tokens are exactly as expected,
+            // partly because that's hard to parameterize.
+            var reader = new StringReader(json.Replace('\'', '"'));
+            var tokenizer = JsonTokenizer.FromTextReader(reader);
+            for (int i = 0; i < expectedValidTokens; i++)
+            {
+                Assert.IsNotNull(tokenizer.Next());
+            }
+            Assert.Throws<InvalidJsonException>(() => tokenizer.Next());
+        }
+
+        [Test]
+        public void ArrayMixedType()
+        {
+            AssertTokens("[1, 'foo', null, false, true, [2], {'x':'y' }]",
+                JsonToken.StartArray,
+                JsonToken.Value(1),
+                JsonToken.Value("foo"),
+                JsonToken.Null,
+                JsonToken.False,
+                JsonToken.True,
+                JsonToken.StartArray,
+                JsonToken.Value(2),
+                JsonToken.EndArray,
+                JsonToken.StartObject,
+                JsonToken.Name("x"),
+                JsonToken.Value("y"),
+                JsonToken.EndObject,
+                JsonToken.EndArray);
+        }
+
+        [Test]
+        public void ObjectMixedType()
+        {
+            AssertTokens(@"{'a': 1, 'b': 'bar', 'c': null, 'd': false, 'e': true, 
+                           'f': [2], 'g': {'x':'y' }}",
+                JsonToken.StartObject,
+                JsonToken.Name("a"),
+                JsonToken.Value(1),
+                JsonToken.Name("b"),
+                JsonToken.Value("bar"),
+                JsonToken.Name("c"),
+                JsonToken.Null,
+                JsonToken.Name("d"),
+                JsonToken.False,
+                JsonToken.Name("e"),
+                JsonToken.True,
+                JsonToken.Name("f"),
+                JsonToken.StartArray,
+                JsonToken.Value(2),
+                JsonToken.EndArray,
+                JsonToken.Name("g"),
+                JsonToken.StartObject,
+                JsonToken.Name("x"),
+                JsonToken.Value("y"),
+                JsonToken.EndObject,
+                JsonToken.EndObject);
+        }
+
+        [Test]
+        public void NextAfterEndDocumentThrows()
+        {
+            var tokenizer = JsonTokenizer.FromTextReader(new StringReader("null"));
+            Assert.AreEqual(JsonToken.Null, tokenizer.Next());
+            Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next());
+            Assert.Throws<InvalidOperationException>(() => tokenizer.Next());
+        }
+
+        [Test]
+        public void CanPushBackEndDocument()
+        {
+            var tokenizer = JsonTokenizer.FromTextReader(new StringReader("null"));
+            Assert.AreEqual(JsonToken.Null, tokenizer.Next());
+            Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next());
+            tokenizer.PushBack(JsonToken.EndDocument);
+            Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next());
+            Assert.Throws<InvalidOperationException>(() => tokenizer.Next());
+        }
+       
+        /// <summary>
+        /// Asserts that the specified JSON is tokenized into the given sequence of tokens.
+        /// All apostrophes are first converted to double quotes, allowing any tests
+        /// that don't need to check actual apostrophe handling to use apostrophes in the JSON, avoiding
+        /// messy string literal escaping. The "end document" token is not specified in the list of 
+        /// expected tokens, but is implicit.
+        /// </summary>
+        private static void AssertTokens(string json, params JsonToken[] expectedTokens)
+        {
+            AssertTokensNoReplacement(json.Replace('\'', '"'), expectedTokens);
+        }
+
+        /// <summary>
+        /// Asserts that the specified JSON is tokenized into the given sequence of tokens.
+        /// Unlike <see cref="AssertTokens(string, JsonToken[])"/>, this does not perform any character
+        /// replacement on the specified JSON, and should be used when the text contains apostrophes which
+        /// are expected to be used *as* apostrophes. The "end document" token is not specified in the list of 
+        /// expected tokens, but is implicit.
+        /// </summary>
+        private static void AssertTokensNoReplacement(string json, params JsonToken[] expectedTokens)
+        {
+            var reader = new StringReader(json);
+            var tokenizer = JsonTokenizer.FromTextReader(reader);
+            for (int i = 0; i < expectedTokens.Length; i++)
+            {
+                var actualToken = tokenizer.Next();
+                if (actualToken == JsonToken.EndDocument)
+                {
+                    Assert.Fail("Expected {0} but reached end of token stream", expectedTokens[i]);
+                }
+                Assert.AreEqual(expectedTokens[i], actualToken);
+            }
+            var finalToken = tokenizer.Next();
+            if (finalToken != JsonToken.EndDocument)
+            {
+                Assert.Fail("Expected token stream to be exhausted; received {0}", finalToken);
+            }
+        }
+
+        private static void AssertThrowsAfter(string json, params JsonToken[] expectedTokens)
+        {
+            var reader = new StringReader(json);
+            var tokenizer = JsonTokenizer.FromTextReader(reader);
+            for (int i = 0; i < expectedTokens.Length; i++)
+            {
+                var actualToken = tokenizer.Next();
+                if (actualToken == JsonToken.EndDocument)
+                {
+                    Assert.Fail("Expected {0} but reached end of document", expectedTokens[i]);
+                }
+                Assert.AreEqual(expectedTokens[i], actualToken);
+            }
+            Assert.Throws<InvalidJsonException>(() => tokenizer.Next());
+        }
+    }
+}