[Merge M-58] Adds ChineseScriptClassifier to predict zh-Hant or zh-Hans for input detected as zh.

components/translate/core/language_detection/language_detection_util.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/translate/core/language_detection/language_detection_util.h"
 
 #include <stddef.h>
 
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/metrics/histogram_base.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/metrics_hashes.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/time/time.h"
 #include "components/translate/core/common/translate_constants.h"
 #include "components/translate/core/common/translate_metrics.h"
 #include "components/translate/core/common/translate_util.h"
+#include "components/translate/core/language_detection/chinese_script_classifier.h"
 #include "third_party/cld/cld_version.h"
 
 #if BUILDFLAG(CLD_VERSION) == 2
 #include "third_party/cld_2/src/public/compact_lang_det.h"
 #include "third_party/cld_2/src/public/encodings.h"
 #elif BUILDFLAG(CLD_VERSION) == 3
 #include "third_party/cld_3/src/src/nnet_language_identifier.h"
 #else
 # error "CLD_VERSION must be 2 or 3"
 #endif
@@ skipping 168 matching lines @@
   // Ignore unreliable, "unknown", and xx-Latn predictions that are currently
   // not supported.
   if (prediction_reliable &&
       predicted_language != "bg-Latn" &&
       predicted_language != "el-Latn" &&
       predicted_language != "ja-Latn" &&
       predicted_language != "ru-Latn" &&
       predicted_language != "zh-Latn" &&
       predicted_language !=
           chrome_lang_id::NNetLanguageIdentifier::kUnknown) {
-    // CLD3 returns 'zh' for Chinese but Translate doesn't accept it. Thus,
-    // analogously to CLD2, 'zh-CN' is returned instead.
-    if (predicted_language == "zh") {
-      language = "zh-CN";
+    if (predicted_language != "zh") {
+      language = predicted_language;
     } else {
-      language = predicted_language;
+      // If prediction is "zh" (Chinese), then we need to determine whether the
+      // text is zh-Hant (Chinese Traditional) or zh-Hans (Chinese Simplified).
+      translate::ChineseScriptClassifier zh_classifier;
+
+      // The Classify function returns either "zh-Hant" or "zh-Hans".
+      // Convert to the old-style language codes used by the Translate API.
+      const std::string zh_classification = zh_classifier.Classify(utf8_text);
+      if (zh_classification == "zh-Hant") {
+        language = "zh-TW";
+      } else if (zh_classification == "zh-Hans") {
+        language = "zh-CN";
+      } else {
+        language = translate::kUnknownLanguageCode;
+      }
     }
   }
-
 #else
 # error "CLD_VERSION must be 2 or 3"
 #endif
 
+  VLOG(1) << "Detected language: " << language;
   return language;
 }
 
 // Checks if CLD can complement a sub code when the page language doesn't know
 // the sub code.
 bool CanCLDComplementSubCode(
     const std::string& page_language, const std::string& cld_language) {
   // Translate server cannot treat general Chinese. If Content-Language and
   // CLD agree that the language is Chinese and Content-Language doesn't know
   // which dialect is used, CLD language has priority.
@@ skipping 192 matching lines @@
   // distinguish from English, and the language is one of well-known languages
   // which often provide "en-*" meta information mistakenly.
   for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) {
     if (cld_language == kWellKnownCodesOnWrongConfiguration[i])
       return true;
   }
   return false;
 }
 
 }  // namespace translate