Add new SplitString backend.

base/strings/string_util.cc

Index: base/strings/string_util.cc
diff --git a/base/strings/string_util.cc b/base/strings/string_util.cc
index cc7769343b5a592a2d081c89e3203dc82be8f720..540fc0948439e18276502d65c1211e1372bc3130 100644
--- a/base/strings/string_util.cc
+++ b/base/strings/string_util.cc
@@ -21,6 +21,7 @@
 #include "base/basictypes.h"
 #include "base/logging.h"
 #include "base/memory/singleton.h"
+#include "base/strings/string_split.h"
 #include "base/strings/utf_string_conversion_utils.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/third_party/icu/icu_utf.h"
@@ -193,29 +194,33 @@ bool RemoveChars(const std::string& input,
   return ReplaceChars(input, remove_chars.as_string(), std::string(), output);
 }
 
-template<typename STR>
-TrimPositions TrimStringT(const STR& input,
-                          const STR& trim_chars,
+template<typename Str>
+TrimPositions TrimStringT(const Str& input,
+                          BasicStringPiece<Str> trim_chars,
                           TrimPositions positions,
-                          STR* output) {
-  // Find the edges of leading/trailing whitespace as desired.
+                          Str* output) {
+  // Find the edges of leading/trailing whitespace as desired. Need to use
+  // a StringPiece version of input to be able to call find* on it with the
+  // StringPiece version of trim_chars (normally the trim_chars will be a
+  // constant so avoid making a copy).
+  BasicStringPiece<Str> input_piece(input);
   const size_t last_char = input.length() - 1;
   const size_t first_good_char = (positions & TRIM_LEADING) ?
-      input.find_first_not_of(trim_chars) : 0;
+      input_piece.find_first_not_of(trim_chars) : 0;
   const size_t last_good_char = (positions & TRIM_TRAILING) ?
-      input.find_last_not_of(trim_chars) : last_char;
+      input_piece.find_last_not_of(trim_chars) : last_char;
 
-  // When the string was all whitespace, report that we stripped off whitespace
-  // from whichever position the caller was interested in.  For empty input, we
-  // stripped no whitespace, but we still need to clear |output|.
+  // When the string was all trimmed, report that we stripped off characters
+  // from whichever position the caller was interested in. For empty input, we
+  // stripped no characters, but we still need to clear |output|.
   if (input.empty() ||
-      (first_good_char == STR::npos) || (last_good_char == STR::npos)) {
+      (first_good_char == Str::npos) || (last_good_char == Str::npos)) {
     bool input_was_empty = input.empty();  // in case output == &input
     output->clear();
     return input_was_empty ? TRIM_NONE : positions;
   }
 
-  // Trim the whitespace.
+  // Trim.
   *output =
       input.substr(first_good_char, last_good_char - first_good_char + 1);
 
@@ -226,17 +231,38 @@ TrimPositions TrimStringT(const STR& input,
 }
 
 bool TrimString(const string16& input,
-                const base::StringPiece16& trim_chars,
+                base::StringPiece16 trim_chars,
                 string16* output) {
-  return TrimStringT(input, trim_chars.as_string(), TRIM_ALL, output) !=
-      TRIM_NONE;
+  return TrimStringT(input, trim_chars, TRIM_ALL, output) != TRIM_NONE;
 }
 
 bool TrimString(const std::string& input,
-                const base::StringPiece& trim_chars,
+                base::StringPiece trim_chars,
                 std::string* output) {
-  return TrimStringT(input, trim_chars.as_string(), TRIM_ALL, output) !=
-      TRIM_NONE;
+  return TrimStringT(input, trim_chars, TRIM_ALL, output) != TRIM_NONE;
+}
+
+template<typename Str>
+BasicStringPiece<Str> TrimStringPieceT(BasicStringPiece<Str> input,
+                                       BasicStringPiece<Str> trim_chars,
+                                       TrimPositions positions) {
+  size_t begin = (positions & TRIM_LEADING) ?
+      input.find_first_not_of(trim_chars) : 0;
+  size_t end = (positions & TRIM_TRAILING) ?
+      input.find_last_not_of(trim_chars) + 1 : input.size();
+  return input.substr(begin, end - begin);
+}
+
+StringPiece16 TrimString(StringPiece16 input,
+                         const base::StringPiece16& trim_chars,
+                         TrimPositions positions) {
+  return TrimStringPieceT(input, trim_chars, positions);
+}
+
+StringPiece TrimString(StringPiece input,
+                       const base::StringPiece& trim_chars,
+                       TrimPositions positions) {
+  return TrimStringPieceT(input, trim_chars, positions);
 }
 
 void TruncateUTF8ToByteSize(const std::string& input,
@@ -278,14 +304,13 @@ void TruncateUTF8ToByteSize(const std::string& input,
 TrimPositions TrimWhitespace(const string16& input,
                              TrimPositions positions,
                              string16* output) {
-  return TrimStringT(input, base::string16(kWhitespaceUTF16), positions,
-                     output);
+  return TrimStringT(input, StringPiece16(kWhitespaceUTF16), positions, output);
 }
 
 TrimPositions TrimWhitespaceASCII(const std::string& input,
                                   TrimPositions positions,
                                   std::string* output) {
-  return TrimStringT(input, std::string(kWhitespaceASCII), positions, output);
+  return TrimStringT(input, StringPiece(kWhitespaceASCII), positions, output);
 }
 
 // This function is only for backward-compatibility.
@@ -696,44 +721,28 @@ void ReplaceSubstringsAfterOffset(std::string* str,
                                  true);  // replace all instances
 }
 
-
-template<typename STR>
-static size_t TokenizeT(const STR& str,
-                        const STR& delimiters,
-                        std::vector<STR>* tokens) {
-  tokens->clear();
-
-  size_t start = str.find_first_not_of(delimiters);
-  while (start != STR::npos) {
-    size_t end = str.find_first_of(delimiters, start + 1);
-    if (end == STR::npos) {
-      tokens->push_back(str.substr(start));
-      break;
-    } else {
-      tokens->push_back(str.substr(start, end - start));
-      start = str.find_first_not_of(delimiters, end + 1);
-    }
-  }
-
+size_t Tokenize(const base::string16& str,
+                const base::string16& delimiters,
+                std::vector<base::string16>* tokens) {
+  *tokens = base::SplitString(
+      str, delimiters, base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY);
   return tokens->size();
 }
 
-size_t Tokenize(const string16& str,

[danakj, 2015/06/11 23:58:35]

Are there unit tests of these functions? It's reading to me like they would have
done SPLIT_WANT_ALL. I don't see any skipping of empty tokens?

[brettw, 2015/06/12 17:37:08]

Yes, search for TokenizeTest in string_util_unittest.cc. The old code does
find_first_not_of(delimiters) which has the effect of skipping a sequence of
delimiters, which is the same as skipping empty ones. If Tokenize collapsed
whitespace, the results would be different, but it always keeps whitespace, so
the results are the same.

-                const string16& delimiters,
-                std::vector<string16>* tokens) {
-  return TokenizeT(str, delimiters, tokens);
-}
-
 size_t Tokenize(const std::string& str,
                 const std::string& delimiters,
                 std::vector<std::string>* tokens) {
-  return TokenizeT(str, delimiters, tokens);
+  *tokens = base::SplitString(
+      str, delimiters, base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY);
+  return tokens->size();
 }
 
 size_t Tokenize(const base::StringPiece& str,
                 const base::StringPiece& delimiters,
                 std::vector<base::StringPiece>* tokens) {
-  return TokenizeT(str, delimiters, tokens);
+  *tokens = base::SplitStringPiece(
+      str, delimiters, base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY);
+  return tokens->size();
 }
 
 template<typename STR>