Move string tokenizer to base/strings.

net/http/http_content_disposition.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/http/http_content_disposition.h"
 
 #include "base/base64.h"
 #include "base/i18n/icu_string_conversions.h"
 #include "base/logging.h"
 #include "base/string_util.h"
+#include "base/strings/string_tokenizer.h"
 #include "base/sys_string_conversions.h"
 #include "base/utf_string_conversions.h"
 #include "net/base/net_util.h"
 #include "net/http/http_util.h"
 #include "third_party/icu/public/common/unicode/ucnv.h"
 
 namespace {
 
 enum RFC2047EncodingType {
   Q_ENCODING,
@@ skipping 103 matching lines @@
 
   // RFC 2047 : one of encoding methods supported by Firefox and relatively
   // widely used by web servers.
   // =?charset?<E>?<encoded string>?= where '<E>' is either 'B' or 'Q'.
   // We don't care about the length restriction (72 bytes) because
   // many web servers generate encoded words longer than the limit.
   std::string decoded_word;
   *is_rfc2047 = true;
   int part_index = 0;
   std::string charset;
-  StringTokenizer t(encoded_word, "?");
+  base::StringTokenizer t(encoded_word, "?");
   RFC2047EncodingType enc_type = Q_ENCODING;
   while (*is_rfc2047 && t.GetNext()) {
     std::string part = t.token();
     switch (part_index) {
       case 0:
         if (part != "=") {
           *is_rfc2047 = false;
           break;
         }
         ++part_index;
@@ skipping 84 matching lines @@
 // strings. Non-ASCII strings are interpreted based on |referrer_charset|.
 bool DecodeFilenameValue(const std::string& input,
                          const std::string& referrer_charset,
                          std::string* output,
                          int* parse_result_flags) {
   int current_parse_result_flags = 0;
   std::string decoded_value;
   bool is_previous_token_rfc2047 = true;
 
   // Tokenize with whitespace characters.
-  StringTokenizer t(input, " \t\n\r");
-  t.set_options(StringTokenizer::RETURN_DELIMS);
+  base::StringTokenizer t(input, " \t\n\r");
+  t.set_options(base::StringTokenizer::RETURN_DELIMS);
   while (t.GetNext()) {
     if (t.token_is_delim()) {
       // If the previous non-delimeter token is not RFC2047-encoded,
       // put in a space in its place. Otheriwse, skip over it.
       if (!is_previous_token_rfc2047)
         decoded_value.push_back(' ');
       continue;
     }
     // We don't support a single multibyte character split into
     // adjacent encoded words. Some broken mail clients emit headers
@@ skipping 11 matching lines @@
     *parse_result_flags |= current_parse_result_flags;
   return true;
 }
 
 // Parses the charset and value-chars out of an ext-value string.
 //
 //  ext-value     = charset  "'" [ language ] "'" value-chars
 bool ParseExtValueComponents(const std::string& input,
                              std::string* charset,
                              std::string* value_chars) {
-  StringTokenizer t(input, "'");
-  t.set_options(StringTokenizer::RETURN_DELIMS);
+  base::StringTokenizer t(input, "'");
+  t.set_options(base::StringTokenizer::RETURN_DELIMS);
   std::string temp_charset;
   std::string temp_value;
   int numDelimsSeen = 0;
   while (t.GetNext()) {
     if (t.token_is_delim()) {
       ++numDelimsSeen;
       continue;
     } else {
       switch (numDelimsSeen) {
         case 0:
@@ skipping 162 matching lines @@
 
   if (!ext_filename.empty())
     filename_ = ext_filename;
   else if (!filename.empty())
     filename_ = filename;
   else
     filename_ = name;
 }
 
 }  // namespace net