Pull new version of protobuf sources.

third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc

 // Protocol Buffers - Google's data interchange format
 // Copyright 2008 Google Inc.  All rights reserved.
-// http://code.google.com/p/protobuf/
+// https://developers.google.com/protocol-buffers/
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 // copyright notice, this list of conditions and the following disclaimer
 // in the documentation and/or other materials provided with the
@@ skipping 390 matching lines @@
   // Test that sh-style comments are not ignored by default.
   { "foo # bar\n"
     "baz", {
     { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 },
     { Tokenizer::TYPE_SYMBOL    , "#"  , 0, 4, 5 },
     { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 6, 9 },
     { Tokenizer::TYPE_IDENTIFIER, "baz", 1, 0, 3 },
     { Tokenizer::TYPE_END       , ""   , 1, 3, 3 },
   }},
 
-  // Bytes with the high-order bit set should not be seen as control characters.
-  { "\300", {
-    { Tokenizer::TYPE_SYMBOL, "\300", 0, 0, 1 },
-    { Tokenizer::TYPE_END   , ""    , 0, 1, 1 },
-  }},
-
   // Test all whitespace chars
   { "foo\n\t\r\v\fbar", {
     { Tokenizer::TYPE_IDENTIFIER, "foo", 0,  0,  3 },
     { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11, 14 },
     { Tokenizer::TYPE_END       , ""   , 1, 14, 14 },
   }},
 };
 
 TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {
   // Set up the tokenizer.
@@ skipping 304 matching lines @@
   EXPECT_EQ(0xabcdef12u, ParseInteger("0xabcdef12"));
   EXPECT_EQ(0xabcdef12u, ParseInteger("0xABCDEF12"));
   EXPECT_EQ(kuint64max, ParseInteger("0xFFFFFFFFFFFFFFFF"));
   EXPECT_EQ(01234567, ParseInteger("01234567"));
   EXPECT_EQ(0X123, ParseInteger("0X123"));
 
   // Test invalid integers that may still be tokenized as integers.
   EXPECT_EQ(0, ParseInteger("0x"));
 
   uint64 i;
-#ifdef PROTOBUF_HASDEATH_TEST  // death tests do not work on Windows yet
+#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
   // Test invalid integers that will never be tokenized as integers.
   EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("zxy", kuint64max, &i),
     "passed text that could not have been tokenized as an integer");
   EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("1.2", kuint64max, &i),
     "passed text that could not have been tokenized as an integer");
   EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("08", kuint64max, &i),
     "passed text that could not have been tokenized as an integer");
   EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("0xg", kuint64max, &i),
     "passed text that could not have been tokenized as an integer");
   EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("-1", kuint64max, &i),
     "passed text that could not have been tokenized as an integer");
-#endif  // PROTOBUF_HASDEATH_TEST
+#endif  // PROTOBUF_HAS_DEATH_TEST
 
   // Test overflows.
   EXPECT_TRUE (Tokenizer::ParseInteger("0", 0, &i));
   EXPECT_FALSE(Tokenizer::ParseInteger("1", 0, &i));
   EXPECT_TRUE (Tokenizer::ParseInteger("1", 1, &i));
   EXPECT_TRUE (Tokenizer::ParseInteger("12345", 12345, &i));
   EXPECT_FALSE(Tokenizer::ParseInteger("12346", 12345, &i));
   EXPECT_TRUE (Tokenizer::ParseInteger("0xFFFFFFFFFFFFFFFF" , kuint64max, &i));
   EXPECT_FALSE(Tokenizer::ParseInteger("0x10000000000000000", kuint64max, &i));
 }
@@ skipping 22 matching lines @@
   // Test 'f' suffix.
   EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1f"));
   EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.0f"));
   EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1F"));
 
   // These should parse successfully even though they are out of range.
   // Overflows become infinity and underflows become zero.
   EXPECT_EQ(     0.0, Tokenizer::ParseFloat("1e-9999999999999999999999999999"));
   EXPECT_EQ(HUGE_VAL, Tokenizer::ParseFloat("1e+9999999999999999999999999999"));
 
-#ifdef PROTOBUF_HASDEATH_TEST  // death tests do not work on Windows yet
+#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
   // Test invalid integers that will never be tokenized as integers.
   EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("zxy"),
     "passed text that could not have been tokenized as a float");
   EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("1-e0"),
     "passed text that could not have been tokenized as a float");
   EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("-1.0"),
     "passed text that could not have been tokenized as a float");
-#endif  // PROTOBUF_HASDEATH_TEST
+#endif  // PROTOBUF_HAS_DEATH_TEST
 }
 
 TEST_F(TokenizerTest, ParseString) {
   string output;
   Tokenizer::ParseString("'hello'", &output);
   EXPECT_EQ("hello", output);
   Tokenizer::ParseString("\"blah\\nblah2\"", &output);
   EXPECT_EQ("blah\nblah2", output);
   Tokenizer::ParseString("'\\1x\\1\\123\\739\\52\\334n\\3'", &output);
   EXPECT_EQ("\1x\1\123\739\52\334n\3", output);
@@ skipping 18 matching lines @@
   // Here's some broken UTF16; there's a head surrogate with no tail surrogate.
   // We just output this as if it were UTF8; it's not a defined code point, but
   // it has a defined encoding.
   Tokenizer::ParseString("'\\ud852XX'", &output);
   EXPECT_EQ("\xed\xa1\x92XX", output);
   // Malformed escape: Demons may fly out of the nose.
   Tokenizer::ParseString("\\u0", &output);
   EXPECT_EQ("u0", output);
 
   // Test invalid strings that will never be tokenized as strings.
-#ifdef PROTOBUF_HASDEATH_TEST  // death tests do not work on Windows yet
+#ifdef PROTOBUF_HAS_DEATH_TEST  // death tests do not work on Windows yet
   EXPECT_DEBUG_DEATH(Tokenizer::ParseString("", &output),
     "passed text that could not have been tokenized as a string");
-#endif  // PROTOBUF_HASDEATH_TEST
+#endif  // PROTOBUF_HAS_DEATH_TEST
 }
 
 TEST_F(TokenizerTest, ParseStringAppend) {
   // Check that ParseString and ParseStringAppend differ.
   string output("stuff+");
   Tokenizer::ParseStringAppend("'hello'", &output);
   EXPECT_EQ("stuff+hello", output);
   Tokenizer::ParseString("'hello'", &output);
   EXPECT_EQ("hello", output);
 }
@@ skipping 16 matching lines @@
   return out << CEscape(test_case.input);
 }
 
 ErrorCase kErrorCases[] = {
   // String errors.
   { "'\\l' foo", true,
     "0:2: Invalid escape sequence in string literal.\n" },
   { "'\\x' foo", true,
     "0:3: Expected hex digits for escape sequence.\n" },
   { "'foo", false,
-    "0:4: String literals cannot cross line boundaries.\n" },
+    "0:4: Unexpected end of string.\n" },
   { "'bar\nfoo", true,
     "0:4: String literals cannot cross line boundaries.\n" },
   { "'\\u01' foo", true,
     "0:5: Expected four hex digits for \\u escape sequence.\n" },
   { "'\\u01' foo", true,
     "0:5: Expected four hex digits for \\u escape sequence.\n" },
   { "'\\uXYZ' foo", true,
     "0:3: Expected four hex digits for \\u escape sequence.\n" },
 
   // Integer errors.
@@ skipping 47 matching lines @@
     "0:0: Invalid control characters encountered in text.\n" },
 
   // Check recovery from '\0'.  We have to explicitly specify the length of
   // these strings because otherwise the string constructor will just call
   // strlen() which will see the first '\0' and think that is the end of the
   // string.
   { string("\0foo", 4), true,
     "0:0: Invalid control characters encountered in text.\n" },
   { string("\0\0foo", 5), true,
     "0:0: Invalid control characters encountered in text.\n" },
+
+  // Check error from high order bits set
+  { "\300foo", true,
+    "0:0: Interpreting non ascii codepoint 192.\n" },
 };
 
 TEST_2D(TokenizerTest, Errors, kErrorCases, kBlockSizes) {
   // Set up the tokenizer.
   TestInputStream input(kErrorCases_case.input.data(),
                         kErrorCases_case.input.size(),
                         kBlockSizes_case);
   TestErrorCollector error_collector;
   Tokenizer tokenizer(&input, &error_collector);
 
@@ skipping 28 matching lines @@
 
   // Only "foo" should have been read.
   EXPECT_EQ(strlen("foo"), input.ByteCount());
 }
 
 
 }  // namespace
 }  // namespace io
 }  // namespace protobuf
 }  // namespace google