FTP: improve character encoding detection in cases where ICU's first guess is wrong.

base/i18n/icu_encoding_detection.cc

Index: base/i18n/icu_encoding_detection.cc
diff --git a/base/i18n/icu_encoding_detection.cc b/base/i18n/icu_encoding_detection.cc
index 55785c51a86ff2eaaed2c4f1a62d7e919aa2750c..d579af280a7af7959ca6d48a3f125c87b98cb13c 100644
--- a/base/i18n/icu_encoding_detection.cc
+++ b/base/i18n/icu_encoding_detection.cc
@@ -9,8 +9,6 @@
 
 namespace base {
 
-// TODO(jungshik): We can apply more heuristics here (e.g. using various hints
-// like TLD, the UI language/default encoding of a client, etc).
 bool DetectEncoding(const std::string& text, std::string* encoding) {
   if (IsStringASCII(text)) {
     *encoding = std::string();
@@ -21,9 +19,6 @@ bool DetectEncoding(const std::string& text, std::string* encoding) {
   UCharsetDetector* detector = ucsdet_open(&status);
   ucsdet_setText(detector, text.data(), static_cast<int32_t>(text.length()),
                  &status);
-  // TODO(jungshik): Should we check the quality of the match? A rather
-  // arbitrary number is assigned by ICU and it's hard to come up with
-  // a lower limit.
   const UCharsetMatch* match = ucsdet_detect(detector, &status);
   const char* detected_encoding = ucsdet_getName(match, &status);
   ucsdet_close(detector);
@@ -35,4 +30,35 @@ bool DetectEncoding(const std::string& text, std::string* encoding) {
   return true;
 }
 
+bool DetectAllEncodings(const std::string& text,

[cbentzel, 2010/11/18 16:39:43]

Is a bool needed here? Could empty encodings be sufficient? 

+                        std::vector<std::string>* encodings) {
+  UErrorCode status = U_ZERO_ERROR;
+  UCharsetDetector* detector = ucsdet_open(&status);

[cbentzel, 2010/11/18 16:39:43]

Should you check for failure after this call? 

http://icu-project.org/apiref/icu4c/ucsdet_8h.html#59f0ebad887fcf9d9259e60a51...

indicates that status is filled with any error conditions.

Main concern is if it does fail, is the return value guaranteed, and would a
ucsdet_close on it lead to issues?

+  ucsdet_setText(detector, text.data(), static_cast<int32_t>(text.length()),

[cbentzel, 2010/11/18 16:39:43]

Should you check for failure after this call as well?

+                 &status);
+  int matches_count = 0;
+  const UCharsetMatch** matches = ucsdet_detectAll(detector,
+                                                   &matches_count,
+                                                   &status);
+  if (U_FAILURE(status)) {
+    ucsdet_close(detector);
+    return false;
+  }
+
+  encodings->clear();
+  for (int i = 0; i < matches_count; i++) {
+    UErrorCode get_name_status = U_ZERO_ERROR;
+    const char* encoding_name = ucsdet_getName(matches[i], &get_name_status);
+
+    // If we failed to get the encoding's name, ignore the error.
+    if (U_FAILURE(get_name_status))
+      continue;
+
+    encodings->push_back(encoding_name);
+  }
+
+  ucsdet_close(detector);
+  return !encodings->empty();
+}
+
 }  // namespace base