Implement CLD hints to CLD2 calls. Edit CLD2 result to return top language instead of summary langu…

components/translate/core/language_detection/language_detection_util.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/translate/core/language_detection/language_detection_util.h"
 
 #include "base/logging.h"
 #include "base/metrics/field_trial.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/time/time.h"
 #include "components/translate/core/common/translate_constants.h"
 #include "components/translate/core/common/translate_metrics.h"
 #include "components/translate/core/common/translate_util.h"
 
 #if !defined(CLD_VERSION) || CLD_VERSION==1
 #include "third_party/cld/encodings/compact_lang_det/compact_lang_det.h"
 #include "third_party/cld/encodings/compact_lang_det/win/cld_unicodetext.h"
 #endif
 
 #if !defined(CLD_VERSION) || CLD_VERSION==2
 #include "third_party/cld_2/src/public/compact_lang_det.h"
+#include "third_party/cld_2/src/public/encodings.h"
 #endif
 
 namespace {
 
 // Similar language code list. Some languages are very similar and difficult
 // for CLD to distinguish.
 struct SimilarLanguageCode {
   const char* const code;
   int group;
 };
@@ skipping 45 matching lines @@
     return 1;
 #else
   return CLD_VERSION;
 #endif
 }
 
 // Returns the ISO 639 language code of the specified |text|, or 'unknown' if it
 // failed.
 // |is_cld_reliable| will be set as true if CLD says the detection is reliable.
 std::string DetermineTextLanguage(const base::string16& text,
-                                  bool* is_cld_reliable) {
+                                  bool* is_cld_reliable,
+                                  std::string& code,
+                                  std::string& html_lang) {
   std::string language = translate::kUnknownLanguageCode;
   int num_bytes_evaluated = 0;
   bool is_reliable = false;
   const bool is_plain_text = true;
 
   // Language or CLD2::Language
   int cld_language = 0;
   bool is_valid_language = false;
 
   switch (GetCLDMajorVersion()) {
 #if !defined(CLD_VERSION) || CLD_VERSION==1
     case 1: {
       int num_languages = 0;
       cld_language = DetectLanguageOfUnicodeText(
           NULL, text.c_str(), is_plain_text, &is_reliable, &num_languages, NULL,
           &num_bytes_evaluated);
       is_valid_language = cld_language != NUM_LANGUAGES &&
           cld_language != UNKNOWN_LANGUAGE &&
           cld_language != TG_UNKNOWN_LANGUAGE;
       break;
     }
 #endif
 #if !defined(CLD_VERSION) || CLD_VERSION==2
     case 2: {
       const std::string utf8_text(base::UTF16ToUTF8(text));
       const int num_utf8_bytes = static_cast<int>(utf8_text.size());
       const char* raw_utf8_bytes = utf8_text.c_str();
-      cld_language = CLD2::DetectLanguageCheckUTF8(
-          raw_utf8_bytes, num_utf8_bytes, is_plain_text, &is_reliable,
-          &num_bytes_evaluated);
+
+      CLD2::Language language3[3];
+      int percent3[3];
+      int flags = 0;   // No flags, see compact_lang_det.h for details.
+      int text_bytes;  // Amount of non-tag/letters-only text (assumed 0).
+      double normalized_score3[3];
+
+      const char* tld_hint = "";
+      int encoding_hint = CLD2::UNKNOWN_ENCODING;
+      CLD2::Language language_hint =
+          CLD2::GetLanguageFromName(html_lang.c_str());
+      CLD2::CLDHints cldhints = {code.c_str(), tld_hint, encoding_hint,
+                                 language_hint};
+
+      cld_language = CLD2::ExtDetectLanguageSummaryCheckUTF8(
+          raw_utf8_bytes, num_utf8_bytes, is_plain_text, &cldhints, flags,
+          language3, percent3, normalized_score3, nullptr, &text_bytes,

[Andrew Hayden (chromium.org), 2015/08/05 11:22:45]

Please comment the meaning of the nullptr here, we typically would do something
like this:

lib::SomeFunction(param1, param2, param3, param4,
  nullptr /* super_special_parameter */, param5,
  param6);

+          &is_reliable, &num_bytes_evaluated);
 
       if (num_bytes_evaluated < num_utf8_bytes &&
           cld_language == CLD2::UNKNOWN_LANGUAGE) {
         // Invalid UTF8 encountered, see bug http://crbug.com/444258.
         // Retry using only the valid characters. This time the check for valid
         // UTF8 can be skipped since the precise number of valid bytes is known.
-        cld_language = CLD2::DetectLanguage(raw_utf8_bytes, num_bytes_evaluated,
-                                            is_plain_text, &is_reliable);
+        cld_language = CLD2::ExtDetectLanguageSummary(
+            raw_utf8_bytes, num_utf8_bytes, is_plain_text, &cldhints, flags,
+            language3, percent3, normalized_score3, nullptr, &text_bytes,

[Andrew Hayden (chromium.org), 2015/08/05 11:22:45]

And same here, please

+            &is_reliable);
       }
       is_valid_language = cld_language != CLD2::NUM_LANGUAGES &&
           cld_language != CLD2::UNKNOWN_LANGUAGE &&
           cld_language != CLD2::TG_UNKNOWN_LANGUAGE;
+
+      // Choose top language.
+      cld_language = language3[0];
       break;
     }
 #endif
     default:
       NOTREACHED();
   }
 
   if (is_cld_reliable != NULL)
     *is_cld_reliable = is_reliable;
 
@@ skipping 64 matching lines @@
 
 namespace translate {
 
 std::string DeterminePageLanguage(const std::string& code,
                                   const std::string& html_lang,
                                   const base::string16& contents,
                                   std::string* cld_language_p,
                                   bool* is_cld_reliable_p) {
   base::TimeTicks begin_time = base::TimeTicks::Now();
   bool is_cld_reliable;
-  std::string cld_language = DetermineTextLanguage(contents, &is_cld_reliable);
-  translate::ReportLanguageDetectionTime(begin_time, base::TimeTicks::Now());
-
-  if (cld_language_p != NULL)
-    *cld_language_p = cld_language;
-  if (is_cld_reliable_p != NULL)
-    *is_cld_reliable_p = is_cld_reliable;
-  translate::ToTranslateLanguageSynonym(&cld_language);
-
   // Check if html lang attribute is valid.
   std::string modified_html_lang;
   if (!html_lang.empty()) {
     modified_html_lang = html_lang;
     ApplyLanguageCodeCorrection(&modified_html_lang);
     translate::ReportHtmlLang(html_lang, modified_html_lang);
     VLOG(9) << "html lang based language code: " << modified_html_lang;
   }
 
   // Check if Content-Language is valid.
   std::string modified_code;
   if (!code.empty()) {
     modified_code = code;
     ApplyLanguageCodeCorrection(&modified_code);
     translate::ReportContentLanguage(code, modified_code);
   }
 
+  std::string cld_language = DetermineTextLanguage(
+      contents, &is_cld_reliable, modified_code, modified_html_lang);
+  translate::ReportLanguageDetectionTime(begin_time, base::TimeTicks::Now());
+
+  if (cld_language_p != NULL)
+    *cld_language_p = cld_language;
+  if (is_cld_reliable_p != NULL)
+    *is_cld_reliable_p = is_cld_reliable;
+  translate::ToTranslateLanguageSynonym(&cld_language);
+
   // Adopt |modified_html_lang| if it is valid. Otherwise, adopt
   // |modified_code|.
   std::string language = modified_html_lang.empty() ? modified_code :
                                                       modified_html_lang;
 
   // If |language| is empty, just use CLD result even though it might be
   // translate::kUnknownLanguageCode.
   if (language.empty()) {
     translate::ReportLanguageVerification(
         translate::LANGUAGE_VERIFICATION_CLD_ONLY);
@@ skipping 138 matching lines @@
   // distinguish from English, and the language is one of well-known languages
   // which often provide "en-*" meta information mistakenly.
   for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) {
     if (cld_language == kWellKnownCodesOnWrongConfiguration[i])
       return true;
   }
   return false;
 }
 
 }  // namespace translate