Implement CLD hints to CLD2 calls. Edit CLD2 result to return top language instead of summary langu…

components/translate/core/language_detection/language_detection_util.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/translate/core/language_detection/language_detection_util.h"
 
 #include "base/logging.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/time/time.h"
 #include "components/translate/core/common/translate_constants.h"
 #include "components/translate/core/common/translate_metrics.h"
 #include "components/translate/core/common/translate_util.h"
 
 #if CLD_VERSION==1
 #include "third_party/cld/encodings/compact_lang_det/compact_lang_det.h"
 #include "third_party/cld/encodings/compact_lang_det/win/cld_unicodetext.h"
 #endif
 
 #if CLD_VERSION==2
 #include "third_party/cld_2/src/public/compact_lang_det.h"
+#include "third_party/cld_2/src/public/encodings.h"
 #endif
 
 namespace {
 
 // Similar language code list. Some languages are very similar and difficult
 // for CLD to distinguish.
 struct SimilarLanguageCode {
   const char* const code;
   int group;
 };
@@ skipping 33 matching lines @@
     return;
   }
 
   translate::ToTranslateLanguageSynonym(code);
 }
 
 // Returns the ISO 639 language code of the specified |text|, or 'unknown' if it
 // failed.
 // |is_cld_reliable| will be set as true if CLD says the detection is reliable.
 std::string DetermineTextLanguage(const base::string16& text,
-                                  bool* is_cld_reliable) {
+                                  bool* is_cld_reliable,
+                                  std::string& code,
+                                  std::string& html_lang) {
   std::string language = translate::kUnknownLanguageCode;
   int num_bytes_evaluated = 0;
   bool is_reliable = false;
   const bool is_plain_text = true;
 
   // Language or CLD2::Language
   int cld_language = 0;
   bool is_valid_language = false;
 
 #if CLD_VERSION==1
   int num_languages = 0;
-  cld_language = DetectLanguageOfUnicodeText(
-      NULL, text.c_str(), is_plain_text, &is_reliable, &num_languages, NULL,
-      &num_bytes_evaluated);
+  cld_language = DetectLanguageOfUnicodeText(NULL, text.c_str(), is_plain_text,
+                                             &is_reliable, &num_languages, NULL,
+                                             &num_bytes_evaluated);
   is_valid_language = cld_language != NUM_LANGUAGES &&
-      cld_language != UNKNOWN_LANGUAGE &&
-      cld_language != TG_UNKNOWN_LANGUAGE;
+                      cld_language != UNKNOWN_LANGUAGE &&
+                      cld_language != TG_UNKNOWN_LANGUAGE;
 #elif CLD_VERSION==2
   const std::string utf8_text(base::UTF16ToUTF8(text));
   const int num_utf8_bytes = static_cast<int>(utf8_text.size());
   const char* raw_utf8_bytes = utf8_text.c_str();
-  cld_language = CLD2::DetectLanguageCheckUTF8(
-      raw_utf8_bytes, num_utf8_bytes, is_plain_text, &is_reliable,
+
+  CLD2::Language language3[3];
+  int percent3[3];
+  int flags = 0;   // No flags, see compact_lang_det.h for details.
+  int text_bytes;  // Amount of non-tag/letters-only text (assumed 0).
+  double normalized_score3[3];
+
+  const char* tld_hint = "";
+  int encoding_hint = CLD2::UNKNOWN_ENCODING;
+  CLD2::Language language_hint = CLD2::GetLanguageFromName(html_lang.c_str());
+  CLD2::CLDHints cldhints = {code.c_str(), tld_hint, encoding_hint,
+                             language_hint};
+
+  cld_language = CLD2::ExtDetectLanguageSummaryCheckUTF8(
+      raw_utf8_bytes, num_utf8_bytes, is_plain_text, &cldhints, flags,
+      language3, percent3, normalized_score3,
+      nullptr /* No ResultChunkVector used */, &text_bytes, &is_reliable,
       &num_bytes_evaluated);
 
   if (num_bytes_evaluated < num_utf8_bytes &&
       cld_language == CLD2::UNKNOWN_LANGUAGE) {
     // Invalid UTF8 encountered, see bug http://crbug.com/444258.
     // Retry using only the valid characters. This time the check for valid
     // UTF8 can be skipped since the precise number of valid bytes is known.
-    cld_language = CLD2::DetectLanguage(raw_utf8_bytes, num_bytes_evaluated,
-                                        is_plain_text, &is_reliable);
+    cld_language = CLD2::ExtDetectLanguageSummary(
+        raw_utf8_bytes, num_utf8_bytes, is_plain_text, &cldhints, flags,
+        language3, percent3, normalized_score3,
+        nullptr /* No ResultChunkVector used */, &text_bytes, &is_reliable);
   }
   is_valid_language = cld_language != CLD2::NUM_LANGUAGES &&
-      cld_language != CLD2::UNKNOWN_LANGUAGE &&
-      cld_language != CLD2::TG_UNKNOWN_LANGUAGE;
+                      cld_language != CLD2::UNKNOWN_LANGUAGE &&
+                      cld_language != CLD2::TG_UNKNOWN_LANGUAGE;
+
+  // Choose top language.
+  cld_language = language3[0];
 #else
 # error "CLD_VERSION must be 1 or 2"
 #endif
 
   if (is_cld_reliable != NULL)
     *is_cld_reliable = is_reliable;
 
   // We don't trust the result if the CLD reports that the detection is not
   // reliable, or if the actual text used to detect the language was less than
   // 100 bytes (short texts can often lead to wrong results).
@@ skipping 51 matching lines @@
 
 namespace translate {
 
 std::string DeterminePageLanguage(const std::string& code,
                                   const std::string& html_lang,
                                   const base::string16& contents,
                                   std::string* cld_language_p,
                                   bool* is_cld_reliable_p) {
   base::TimeTicks begin_time = base::TimeTicks::Now();
   bool is_cld_reliable;
-  std::string cld_language = DetermineTextLanguage(contents, &is_cld_reliable);
-  translate::ReportLanguageDetectionTime(begin_time, base::TimeTicks::Now());
-
-  if (cld_language_p != NULL)
-    *cld_language_p = cld_language;
-  if (is_cld_reliable_p != NULL)
-    *is_cld_reliable_p = is_cld_reliable;
-  translate::ToTranslateLanguageSynonym(&cld_language);
-
   // Check if html lang attribute is valid.
   std::string modified_html_lang;
   if (!html_lang.empty()) {
     modified_html_lang = html_lang;
     ApplyLanguageCodeCorrection(&modified_html_lang);
     translate::ReportHtmlLang(html_lang, modified_html_lang);
     VLOG(9) << "html lang based language code: " << modified_html_lang;
   }
 
   // Check if Content-Language is valid.
   std::string modified_code;
   if (!code.empty()) {
     modified_code = code;
     ApplyLanguageCodeCorrection(&modified_code);
     translate::ReportContentLanguage(code, modified_code);
   }
 
+  std::string cld_language = DetermineTextLanguage(
+      contents, &is_cld_reliable, modified_code, modified_html_lang);
+  translate::ReportLanguageDetectionTime(begin_time, base::TimeTicks::Now());
+
+  if (cld_language_p != NULL)
+    *cld_language_p = cld_language;
+  if (is_cld_reliable_p != NULL)
+    *is_cld_reliable_p = is_cld_reliable;
+  translate::ToTranslateLanguageSynonym(&cld_language);
+
   // Adopt |modified_html_lang| if it is valid. Otherwise, adopt
   // |modified_code|.
   std::string language = modified_html_lang.empty() ? modified_code :
                                                       modified_html_lang;
 
   // If |language| is empty, just use CLD result even though it might be
   // translate::kUnknownLanguageCode.
   if (language.empty()) {
     translate::ReportLanguageVerification(
         translate::LANGUAGE_VERIFICATION_CLD_ONLY);
@@ skipping 138 matching lines @@
   // distinguish from English, and the language is one of well-known languages
   // which often provide "en-*" meta information mistakenly.
   for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) {
     if (cld_language == kWellKnownCodesOnWrongConfiguration[i])
       return true;
   }
   return false;
 }
 
 }  // namespace translate