Move basic string operations to ui/app_list/search

chrome/browser/ui/app_list/search/tokenized_string.cc

-// Copyright 2013 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "chrome/browser/ui/app_list/search/tokenized_string.h"
-
-#include "base/i18n/break_iterator.h"
-#include "base/i18n/case_conversion.h"
-#include "base/logging.h"
-#include "chrome/browser/ui/app_list/search/term_break_iterator.h"
-
-using base::i18n::BreakIterator;
-
-namespace app_list {
-
-TokenizedString::TokenizedString(const base::string16& text)
-    : text_(text) {
-  Tokenize();
-}
-
-TokenizedString::~TokenizedString() {}
-
-void TokenizedString::Tokenize() {
-  BreakIterator break_iter(text_,  BreakIterator::BREAK_WORD);
-  if (!break_iter.Init()) {
-    NOTREACHED() << "BreakIterator init failed"
-                 << ", text=\"" << text_ << "\"";
-    return;
-  }
-
-  while (break_iter.Advance()) {
-    if (!break_iter.IsWord())
-      continue;
-
-    const base::string16 word(break_iter.GetString());
-    const size_t word_start = break_iter.prev();
-    TermBreakIterator term_iter(word);
-    while (term_iter.Advance()) {
-      tokens_.push_back(base::i18n::ToLower(term_iter.GetCurrentTerm()));
-      mappings_.push_back(gfx::Range(word_start + term_iter.prev(),
-                                    word_start + term_iter.pos()));
-    }
-  }
-}
-
-}  // namespace app_list