Add support to StringTokenizer for using braces/brackets as quote chars.

base/strings/string_tokenizer.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef BASE_STRINGS_STRING_TOKENIZER_H_
 #define BASE_STRINGS_STRING_TOKENIZER_H_
 
 #include <algorithm>
 #include <string>
 
@@ skipping 40 matching lines @@
 //   }
 //
 // Output:
 //
 //   no-cache="foo, bar"
 //   private
 //
 //
 // EXAMPLE 3:
 //
+//   std::string input = "<foo, bar>, \"bar, foo\", private";
+//   StringTokenizer t(input, ", ");
+//   t.set_paired_quote_chars("\"<", "\">");
+//   while (t.GetNext()) {
+//     printf("%s\n", t.token().c_str());
+//   }
+//
+// Output:
+//
+//   <foo, bar>
+//   "bar, foo"
+//   private
+//
+//
+// EXAMPLE 4:
+//
 //   bool next_is_option = false, next_is_value = false;
 //   std::string input = "text/html; charset=UTF-8; foo=bar";
 //   StringTokenizer t(input, "; =");
 //   t.set_options(StringTokenizer::RETURN_DELIMS);
 //   while (t.GetNext()) {
 //     if (t.token_is_delim()) {
 //       switch (*t.token_begin()) {
 //         case ';':
 //           next_is_option = true;
 //           break;
@@ skipping 42 matching lines @@
   }
 
   // Set the options for this tokenizer.  By default, this is 0.
   void set_options(int options) { options_ = options; }
 
   // Set the characters to regard as quotes.  By default, this is empty.  When
   // a quote char is encountered, the tokenizer will switch into a mode where
   // it ignores delimiters that it finds.  It switches out of this mode once it
   // finds another instance of the quote char.  If a backslash is encountered
   // within a quoted string, then the next character is skipped.
-  void set_quote_chars(const str& quotes) { quotes_ = quotes; }
+  void set_quote_chars(const str& quotes) {
+    set_paired_quote_chars(quotes, quotes);
+  }
+
+  // Similar to set_quote_chars, but makes it possible to have a different char
+  // to start and end a quoted string.  The open_quotes and close_quotes
+  // parameters must have the same number of characters.
+  void set_paired_quote_chars(const str& open_quotes, const str& close_quotes) {
+    DCHECK_EQ(open_quotes.size(), close_quotes.size());
+    open_quotes_ = open_quotes;
+    close_quotes_ = close_quotes;
+  }
 
   // Call this method to advance the tokenizer to the next delimiter.  This
   // returns false if the tokenizer is complete.  This method must be called
   // before calling any of the token* methods.
   bool GetNext() {
-    if (quotes_.empty() && options_ == 0)
+    if (open_quotes_.empty() && options_ == 0)
       return QuickGetNext();
     else
       return FullGetNext();
   }
 
   // Start iterating through tokens from the beginning of the string.
   void Reset() {
     token_end_ = start_pos_;
   }
 
@@ skipping 63 matching lines @@
     }
     while (token_end_ != end_ && AdvanceOne(&state, *token_end_))
       ++token_end_;
     return true;
   }
 
   bool IsDelim(char_type c) const {
     return delims_.find(c) != str::npos;
   }
 
-  bool IsQuote(char_type c) const {
-    return quotes_.find(c) != str::npos;
+  bool IsQuote(char_type c, char_type* close_quote) const {
+    size_t pos = open_quotes_.find(c);
+    if (pos != str::npos)
+      *close_quote = close_quotes_[pos];
+    return pos != str::npos;
   }
 
   struct AdvanceState {
     bool in_quote;
     bool in_escape;
     char_type quote_char;
     AdvanceState() : in_quote(false), in_escape(false), quote_char('\0') {}
   };
 
   // Returns true if a delimiter was not hit.
   bool AdvanceOne(AdvanceState* state, char_type c) {
     if (state->in_quote) {
       if (state->in_escape) {
         state->in_escape = false;
       } else if (c == '\\') {
         state->in_escape = true;
       } else if (c == state->quote_char) {
         state->in_quote = false;
       }
     } else {
       if (IsDelim(c))
         return false;
-      state->in_quote = IsQuote(state->quote_char = c);
+      state->in_quote = IsQuote(c, &state->quote_char);
     }
     return true;
   }
 
   const_iterator start_pos_;
   const_iterator token_begin_;
   const_iterator token_end_;
   const_iterator end_;
   str delims_;
-  str quotes_;
+  str open_quotes_;
+  str close_quotes_;
   int options_;
   bool token_is_delim_;
 };
 
 typedef StringTokenizerT<std::string, std::string::const_iterator>
     StringTokenizer;
 typedef StringTokenizerT<std::wstring, std::wstring::const_iterator>
     WStringTokenizer;
 typedef StringTokenizerT<std::string, const char*> CStringTokenizer;
 
 }  // namespace base
 
 #endif  // BASE_STRINGS_STRING_TOKENIZER_H_