Revert 49594 - Convert page contents grabbing from wide to UTF16. The current...

chrome/renderer/render_view.cc

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/render_view.h"
 
 #include <algorithm>
 #include <string>
 #include <vector>
 
@@ skipping 337 matching lines @@
   std::string new_extension =
       RenderThread::current()->GetExtensionIdForURL(new_url);
   return (old_extension != new_extension);
 }
 
 // Returns the ISO 639_1 language code of the specified |text|, or 'unknown'
 // if it failed.
 //
 // Note this only works on Windows at this time.  It always returns 'unknown'
 // on other platforms.
-static std::string DetermineTextLanguage(const string16& text) {
+static std::string DetermineTextLanguage(const std::wstring& text) {
   // Text with less than 100 bytes will probably not provide good results.
   // Report it as unknown language.
   if (text.length() < 100)
     return RenderView::kUnknownLanguageCode;
 
   std::string language = RenderView::kUnknownLanguageCode;
   int num_languages = 0;
   bool is_reliable = false;
+  string16 input = WideToUTF16(text);
   Language cld_language =
-      DetectLanguageOfUnicodeText(NULL, text.c_str(), true, &is_reliable,
+      DetectLanguageOfUnicodeText(NULL, input.c_str(), true, &is_reliable,
                                   &num_languages, NULL);
   if (is_reliable && cld_language != NUM_LANGUAGES &&
       cld_language != UNKNOWN_LANGUAGE && cld_language != TG_UNKNOWN_LANGUAGE) {
     // We should not use LanguageCode_ISO_639_1 because it does not cover all
     // the languages CLD can detect. As a result, it'll return the invalid
     // language code for tradtional Chinese among others.
     // |LanguageCodeWithDialect| will go through ISO 639-1, ISO-639-2 and
     // 'other' tables to do the 'right' thing. In addition, it'll return zh-CN
     // for Simplified Chinese.
     language = LanguageCodeWithDialects(cld_language);
@@ skipping 441 matching lines @@
 
   if (!preliminary_capture)
     last_indexed_page_id_ = load_id;
 
   // Get the URL for this page.
   GURL url(main_frame->url());
   if (url.is_empty())
     return;
 
   // Retrieve the frame's full text.
-  string16 contents;
+  std::wstring contents;
   CaptureText(main_frame, &contents);
   if (contents.size()) {
     base::TimeTicks begin_time = base::TimeTicks::Now();
     std::string language = DetermineTextLanguage(contents);
     UMA_HISTOGRAM_MEDIUM_TIMES("Renderer4.LanguageDetection",
                                base::TimeTicks::Now() - begin_time);
 
     // Send the text to the browser for indexing (the browser might decide not
     // to index, if the URL is HTTPS for instance) and language discovery.
     Send(new ViewHostMsg_PageContents(routing_id_, url, load_id, contents,
                                       language));
   }
 
   OnCaptureThumbnail();
 }
 
-void RenderView::CaptureText(WebFrame* frame, string16* contents) {
+void RenderView::CaptureText(WebFrame* frame, std::wstring* contents) {
   contents->clear();
   if (!frame)
     return;
 
 #ifdef TIME_TEXT_RETRIEVAL
   double begin = time_util::GetHighResolutionTimeNow();
 #endif
 
   // get the contents of the frame
-  *contents = frame->contentAsText(kMaxIndexChars);
+  *contents = UTF16ToWideHack(frame->contentAsText(kMaxIndexChars));
 
 #ifdef TIME_TEXT_RETRIEVAL
   double end = time_util::GetHighResolutionTimeNow();
   char buf[128];
   sprintf_s(buf, "%d chars retrieved for indexing in %gms\n",
             contents.size(), (end - begin)*1000);
   OutputDebugStringA(buf);
 #endif
 
   // When the contents are clipped to the maximum, we don't want to have a
   // partial word indexed at the end that might have been clipped. Therefore,
   // terminate the string at the last space to ensure no words are clipped.
   if (contents->size() == kMaxIndexChars) {
-    size_t last_space_index = contents->find_last_of(kWhitespaceUTF16);
+    size_t last_space_index = contents->find_last_of(kWhitespaceWide);
     if (last_space_index == std::wstring::npos)
       return;  // don't index if we got a huge block of text with no spaces
     contents->resize(last_space_index);
   }
 }
 
 bool RenderView::CaptureThumbnail(WebView* view,
                                   int w,
                                   int h,
                                   SkBitmap* thumbnail,
@@ skipping 4263 matching lines @@
   webkit_glue::FormData form;
   const WebInputElement element = node.toConst<WebInputElement>();
   if (!form_manager_.FindFormWithFormControlElement(
           element, FormManager::REQUIRE_NONE, &form))
     return;
 
   autofill_action_ = action;
   Send(new ViewHostMsg_FillAutoFillFormData(
       routing_id_, autofill_query_id_, form, value, label));
 }