Add support to StringTokenizer for using braces/brackets as quote chars.

base/strings/string_tokenizer_unittest.cc

Index: base/strings/string_tokenizer_unittest.cc
diff --git a/base/strings/string_tokenizer_unittest.cc b/base/strings/string_tokenizer_unittest.cc
index d391845328291eb8cfe2ad829be740df0b839006..976097210fd5b3244e2a2d403f8b6f83d3dc948c 100644
--- a/base/strings/string_tokenizer_unittest.cc
+++ b/base/strings/string_tokenizer_unittest.cc
@@ -229,6 +229,85 @@ TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) {
   EXPECT_FALSE(t.GetNext());
 }
 
+TEST(StringTokenizerTest, ParsePairQuotedString) {
+  string input = "foo bar <hello world> baz";
+  StringTokenizer t(input, " ");
+  t.set_paired_quote_chars("<", ">");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("foo"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("bar"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("<hello world>"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("baz"), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
+TEST(StringTokenizerTest, ParsePairQuotedString_Malformed) {
+  string input = "bar (hello wo";
+  StringTokenizer t(input, " ");
+  t.set_paired_quote_chars("(", ")");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("bar"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("(hello wo"), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
+TEST(StringTokenizerTest, ParsePairQuotedString_Multiple) {
+  string input = "bar <hel\"lo\" wo> baz\"";
+  StringTokenizer t(input, " ");
+  t.set_paired_quote_chars("'\"<", "'\">");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("bar"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("<hel\"lo\" wo>"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("baz\""), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
+TEST(StringTokenizerTest, ParsePairQuotedString_EscapedQuotes) {
+  string input = "foo <3\\<4>";
+  StringTokenizer t(input, " ");
+  t.set_paired_quote_chars("<", ">");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("foo"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("<3\\<4>"), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
+TEST(StringTokenizerTest, ParsePairQuotedString_EscapedQuotes2) {
+  string input = "foo=<a, b>, bar";
+  StringTokenizer t(input, ", ");
+  t.set_paired_quote_chars("<", ">");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("foo=<a, b>"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("bar"), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
 }  // namespace
 
 }  // namespace base