Add support to StringTokenizer for using braces/brackets as quote chars.

base/strings/string_tokenizer.h

Index: base/strings/string_tokenizer.h
diff --git a/base/strings/string_tokenizer.h b/base/strings/string_tokenizer.h
index 8defbac3b8d0779fbceb6f9744c394428249d482..6afa53a869d99d54a5781aa8a4386d9954abca5b 100644
--- a/base/strings/string_tokenizer.h
+++ b/base/strings/string_tokenizer.h
@@ -58,6 +58,22 @@ namespace base {
 //
 // EXAMPLE 3:
 //
+//   std::string input = "<foo, bar>, \"bar, foo\", private";
+//   StringTokenizer t(input, ", ");
+//   t.set_paired_quote_chars("\"<", "\">");
+//   while (t.GetNext()) {
+//     printf("%s\n", t.token().c_str());
+//   }
+//
+// Output:
+//
+//   <foo, bar>
+//   "bar, foo"
+//   private
+//
+//
+// EXAMPLE 4:
+//
 //   bool next_is_option = false, next_is_value = false;
 //   std::string input = "text/html; charset=UTF-8; foo=bar";
 //   StringTokenizer t(input, "; =");
@@ -120,13 +136,24 @@ class StringTokenizerT {
   // it ignores delimiters that it finds.  It switches out of this mode once it
   // finds another instance of the quote char.  If a backslash is encountered
   // within a quoted string, then the next character is skipped.
-  void set_quote_chars(const str& quotes) { quotes_ = quotes; }
+  void set_quote_chars(const str& quotes) {
+    set_paired_quote_chars(quotes, quotes);
+  }
+
+  // Similar to set_quote_chars, but makes it possible to have a different char
+  // to start and end a quoted string.  The open_quotes and close_quotes
+  // parameters must have the same number of characters.
+  void set_paired_quote_chars(const str& open_quotes, const str& close_quotes) {
+    DCHECK_EQ(open_quotes.size(), close_quotes.size());
+    open_quotes_ = open_quotes;
+    close_quotes_ = close_quotes;
+  }
 
   // Call this method to advance the tokenizer to the next delimiter.  This
   // returns false if the tokenizer is complete.  This method must be called
   // before calling any of the token* methods.
   bool GetNext() {
-    if (quotes_.empty() && options_ == 0)
+    if (open_quotes_.empty() && options_ == 0)
       return QuickGetNext();
     else
       return FullGetNext();
@@ -210,8 +237,11 @@ class StringTokenizerT {
     return delims_.find(c) != str::npos;
   }
 
-  bool IsQuote(char_type c) const {
-    return quotes_.find(c) != str::npos;
+  bool IsQuote(char_type c, char_type* close_quote) const {
+    size_t pos = open_quotes_.find(c);
+    if (pos != str::npos)
+      *close_quote = close_quotes_[pos];
+    return pos != str::npos;
   }
 
   struct AdvanceState {
@@ -234,7 +264,7 @@ class StringTokenizerT {
     } else {
       if (IsDelim(c))
         return false;
-      state->in_quote = IsQuote(state->quote_char = c);
+      state->in_quote = IsQuote(c, &state->quote_char);
     }
     return true;
   }
@@ -244,7 +274,8 @@ class StringTokenizerT {
   const_iterator token_end_;
   const_iterator end_;
   str delims_;
-  str quotes_;
+  str open_quotes_;
+  str close_quotes_;
   int options_;
   bool token_is_delim_;
 };