Add support to StringTokenizer for using braces/brackets as quote chars.

base/strings/string_tokenizer_unittest.cc

 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/strings/string_tokenizer.h"
 
 #include "testing/gtest/include/gtest/gtest.h"
 
 using std::string;
 
@@ skipping 211 matching lines @@
 
   EXPECT_TRUE(t.GetNext());
   EXPECT_EQ(string("foo='a, b'"), t.token());
 
   EXPECT_TRUE(t.GetNext());
   EXPECT_EQ(string("bar"), t.token());
 
   EXPECT_FALSE(t.GetNext());
 }
 
+TEST(StringTokenizerTest, ParsePairQuotedString) {
+  string input = "foo bar <hello world> baz";
+  StringTokenizer t(input, " ");
+  t.set_paired_quote_chars("<", ">");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("foo"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("bar"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("<hello world>"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("baz"), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
+TEST(StringTokenizerTest, ParsePairQuotedString_Malformed) {
+  string input = "bar (hello wo";
+  StringTokenizer t(input, " ");
+  t.set_paired_quote_chars("(", ")");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("bar"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("(hello wo"), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
+TEST(StringTokenizerTest, ParsePairQuotedString_Multiple) {
+  string input = "bar <hel\"lo\" wo> baz\"";
+  StringTokenizer t(input, " ");
+  t.set_paired_quote_chars("'\"<", "'\">");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("bar"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("<hel\"lo\" wo>"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("baz\""), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
+TEST(StringTokenizerTest, ParsePairQuotedString_EscapedQuotes) {
+  string input = "foo <3\\<4>";
+  StringTokenizer t(input, " ");
+  t.set_paired_quote_chars("<", ">");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("foo"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("<3\\<4>"), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
+TEST(StringTokenizerTest, ParsePairQuotedString_EscapedQuotes2) {
+  string input = "foo=<a, b>, bar";
+  StringTokenizer t(input, ", ");
+  t.set_paired_quote_chars("<", ">");
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("foo=<a, b>"), t.token());
+
+  EXPECT_TRUE(t.GetNext());
+  EXPECT_EQ(string("bar"), t.token());
+
+  EXPECT_FALSE(t.GetNext());
+}
+
 }  // namespace
 
 }  // namespace base