Switch language detection to use CLD2's DetectLanguageCheckUTF8 method.

components/translate/core/language_detection/language_detection_util.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/translate/core/language_detection/language_detection_util.h"
 
 #include "base/logging.h"
 #include "base/metrics/field_trial.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
@@ skipping 70 matching lines @@
   return CLD_VERSION;
 #endif
 }
 
 // Returns the ISO 639 language code of the specified |text|, or 'unknown' if it
 // failed.
 // |is_cld_reliable| will be set as true if CLD says the detection is reliable.
 std::string DetermineTextLanguage(const base::string16& text,
                                   bool* is_cld_reliable) {
   std::string language = translate::kUnknownLanguageCode;
-  int text_bytes = 0;
+  int num_bytes_evaluated = 0;
   bool is_reliable = false;
+  const bool is_plain_text = true;
 
   // Language or CLD2::Language
   int cld_language = 0;
   bool is_valid_language = false;
 
   switch (GetCLDMajorVersion()) {
 #if !defined(CLD_VERSION) || CLD_VERSION==1
     case 1: {
       int num_languages = 0;
-      cld_language =
-          DetectLanguageOfUnicodeText(NULL, text.c_str(), true, &is_reliable,
-                                      &num_languages, NULL, &text_bytes);
+      cld_language = DetectLanguageOfUnicodeText(
+          NULL, text.c_str(), is_plain_text, &is_reliable, &num_languages, NULL,
+          &num_bytes_evaluated);
       is_valid_language = cld_language != NUM_LANGUAGES &&
           cld_language != UNKNOWN_LANGUAGE &&
           cld_language != TG_UNKNOWN_LANGUAGE;
       break;
     }
 #endif
 #if !defined(CLD_VERSION) || CLD_VERSION==2
     case 2: {
-      std::string utf8_text(base::UTF16ToUTF8(text));
-      CLD2::Language language3[3];
-      int percent3[3];
-      CLD2::DetectLanguageSummary(
-          utf8_text.c_str(), (int)utf8_text.size(), true, language3, percent3,
-          &text_bytes, &is_reliable);
-      cld_language = language3[0];
+      const std::string utf8_text(base::UTF16ToUTF8(text));
+      const int num_utf8_bytes = static_cast<int>(utf8_text.size());
+      const char* raw_utf8_bytes = utf8_text.c_str();
+      cld_language = CLD2::DetectLanguageCheckUTF8(
+          raw_utf8_bytes, num_utf8_bytes, is_plain_text, &is_reliable,
+          &num_bytes_evaluated);
+
+      if (num_bytes_evaluated < num_utf8_bytes &&
+          cld_language == CLD2::UNKNOWN_LANGUAGE) {
+        // Invalid UTF8 encountered, see bug http://crbug.com/444258.
+        // Retry using only the valid characters. This time the check for valid
+        // UTF8 can be skipped since the precise number of valid bytes is known.
+        cld_language = CLD2::DetectLanguage(raw_utf8_bytes, num_bytes_evaluated,
+                                            is_plain_text, &is_reliable);
+      }
       is_valid_language = cld_language != CLD2::NUM_LANGUAGES &&
           cld_language != CLD2::UNKNOWN_LANGUAGE &&
           cld_language != CLD2::TG_UNKNOWN_LANGUAGE;
       break;
     }
 #endif
     default:
       NOTREACHED();
   }
 
   if (is_cld_reliable != NULL)
     *is_cld_reliable = is_reliable;
 
   // We don't trust the result if the CLD reports that the detection is not
   // reliable, or if the actual text used to detect the language was less than
   // 100 bytes (short texts can often lead to wrong results).
   // TODO(toyoshim): CLD provides |is_reliable| flag. But, it just says that
   // the determined language code is correct with 50% confidence. Chrome should
   // handle the real confidence value to judge.
-  if (is_reliable && text_bytes >= 100 && is_valid_language) {
+  if (is_reliable && num_bytes_evaluated >= 100 && is_valid_language) {
     // We should not use LanguageCode_ISO_639_1 because it does not cover all
     // the languages CLD can detect. As a result, it'll return the invalid
     // language code for tradtional Chinese among others.
     // |LanguageCodeWithDialect| will go through ISO 639-1, ISO-639-2 and
     // 'other' tables to do the 'right' thing. In addition, it'll return zh-CN
     // for Simplified Chinese.
     switch (GetCLDMajorVersion()) {
 #if !defined(CLD_VERSION) || CLD_VERSION==1
       case 1:
         language =
@@ skipping 230 matching lines @@
   // distinguish from English, and the language is one of well-known languages
   // which often provide "en-*" meta information mistakenly.
   for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) {
     if (cld_language == kWellKnownCodesOnWrongConfiguration[i])
       return true;
   }
   return false;
 }
 
 }  // namespace translate