Move string tokenizer to base/strings.

net/http/http_util.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // The rules for parsing content-types were borrowed from Firefox:
 // http://lxr.mozilla.org/mozilla/source/netwerk/base/src/nsURLHelper.cpp#834
 
 #include "net/http/http_util.h"
 
 #include <algorithm>
 
 #include "base/basictypes.h"
 #include "base/logging.h"
-#include "base/stringprintf.h"
 #include "base/string_number_conversions.h"
 #include "base/string_piece.h"
 #include "base/string_util.h"
+#include "base/stringprintf.h"
+#include "base/strings/string_tokenizer.h"
 #include "base/time.h"
 
 using std::string;
 
 namespace net {
 
 //-----------------------------------------------------------------------------
 
 // Return the index of the closing quote of the string, if any.
 static size_t FindStringEnd(const string& line, size_t start, char delim) {
@@ skipping 78 matching lines @@
   if (string::npos == type_end)
     type_end = content_type_str.length();
 
   size_t charset_val = 0;
   size_t charset_end = 0;
   bool type_has_charset = false;
 
   // Iterate over parameters
   size_t param_start = content_type_str.find_first_of(';', type_end);
   if (param_start != string::npos) {
-    StringTokenizer tokenizer(begin + param_start, content_type_str.end(),
-                              ";");
+    base::StringTokenizer tokenizer(begin + param_start, content_type_str.end(),
+                                    ";");
     tokenizer.set_quote_chars("\"");
     while (tokenizer.GetNext()) {
       string::const_iterator equals_sign =
           std::find(tokenizer.token_begin(), tokenizer.token_end(), '=');
       if (equals_sign == tokenizer.token_end())
         continue;
 
       string::const_iterator param_name_begin = tokenizer.token_begin();
       string::const_iterator param_name_end = equals_sign;
       TrimLWS(&param_name_begin, &param_name_end);
@@ skipping 465 matching lines @@
 
   // Copy the status line.
   const char* status_line_end = FindStatusLineEnd(input_begin, input_end);
   raw_headers.append(input_begin, status_line_end);
 
   // After the status line, every subsequent line is a header line segment.
   // Should a segment start with LWS, it is a continuation of the previous
   // line's field-value.
 
   // TODO(ericroman): is this too permissive? (delimits on [\r\n]+)
-  CStringTokenizer lines(status_line_end, input_end, "\r\n");
+  base::CStringTokenizer lines(status_line_end, input_end, "\r\n");
 
   // This variable is true when the previous line was continuable.
   bool prev_line_continuable = false;
 
   while (lines.GetNext()) {
     const char* line_begin = lines.token_begin();
     const char* line_end = lines.token_end();
 
     if (prev_line_continuable && IsLWS(*line_begin)) {
       // Join continuation; reduce the leading LWS to a single SP.
@@ skipping 18 matching lines @@
   // them as line breaks.
   raw_headers.erase(std::remove(raw_headers.begin(), raw_headers.end(), '\0'),
                     raw_headers.end());
   std::replace(raw_headers.begin(), raw_headers.end(), '\n', '\0');
 
   return raw_headers;
 }
 
 std::string HttpUtil::ConvertHeadersBackToHTTPResponse(const std::string& str) {
   std::string disassembled_headers;
-  StringTokenizer tokenizer(str, std::string(1, '\0'));
+  base::StringTokenizer tokenizer(str, std::string(1, '\0'));
   while (tokenizer.GetNext()) {
     disassembled_headers.append(tokenizer.token_begin(), tokenizer.token_end());
     disassembled_headers.append("\r\n");
   }
   disassembled_headers.append("\r\n");
 
   return disassembled_headers;
 }
 
 // TODO(jungshik): 1. If the list is 'fr-CA,fr-FR,en,de', we have to add
 // 'fr' after 'fr-CA' with the same q-value as 'fr-CA' because
 // web servers, in general, do not fall back to 'fr' and may end up picking
 // 'en' which has a lower preference than 'fr-CA' and 'fr-FR'.
 // 2. This function assumes that the input is a comma separated list
 // without any whitespace. As long as it comes from the preference and
 // a user does not manually edit the preference file, it's the case. Still,
 // we may have to make it more robust.
 std::string HttpUtil::GenerateAcceptLanguageHeader(
     const std::string& raw_language_list) {
   // We use integers for qvalue and qvalue decrement that are 10 times
   // larger than actual values to avoid a problem with comparing
   // two floating point numbers.
   const unsigned int kQvalueDecrement10 = 2;
   unsigned int qvalue10 = 10;
-  StringTokenizer t(raw_language_list, ",");
+  base::StringTokenizer t(raw_language_list, ",");
   std::string lang_list_with_q;
   while (t.GetNext()) {
     std::string language = t.token();
     if (qvalue10 == 10) {
       // q=1.0 is implicit.
       lang_list_with_q = language;
     } else {
       DCHECK_LT(qvalue10, 10U);
       base::StringAppendF(&lang_list_with_q, ",%s;q=0.%d", language.c_str(),
                           qvalue10);
@@ skipping 242 matching lines @@
       value_is_quoted_ = true;
       // Do not store iterators into this. See declaration of unquoted_value_.
       unquoted_value_ = HttpUtil::Unquote(value_begin_, value_end_);
     }
   }
 
   return true;
 }
 
 }  // namespace net