FTP: Multiple fixes for localized directory listings:

base/i18n/icu_encoding_detection.cc

Index: base/i18n/icu_encoding_detection.cc
diff --git a/base/i18n/icu_encoding_detection.cc b/base/i18n/icu_encoding_detection.cc
index d579af280a7af7959ca6d48a3f125c87b98cb13c..3583fa9c5f7436ef02b61e177bc433b488fdc0e6 100644
--- a/base/i18n/icu_encoding_detection.cc
+++ b/base/i18n/icu_encoding_detection.cc
@@ -1,9 +1,11 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/i18n/icu_encoding_detection.h"
 
+#include <set>
+
 #include "base/string_util.h"
 #include "unicode/ucsdet.h"
 
@@ -45,6 +47,13 @@ bool DetectAllEncodings(const std::string& text,
     return false;
   }
 
+  // ICU has some heuristics for encoding detection, such that the more likely
+  // encodings should be returned first. However, it doesn't always return
+  // all encodings that properly decode |text|, so we'll append more encodings
+  // later. To make that efficient, keep track of encodings sniffed in this
+  // first phase.
+  std::set<std::string> sniffed_encodings;
+
   encodings->clear();
   for (int i = 0; i < matches_count; i++) {
     UErrorCode get_name_status = U_ZERO_ERROR;
@@ -54,8 +63,37 @@ bool DetectAllEncodings(const std::string& text,
     if (U_FAILURE(get_name_status))
       continue;
 
+    int32_t confidence = ucsdet_getConfidence(matches[i], &get_name_status);
+
+    // We also treat this error as non-fatal.
+    if (U_FAILURE(get_name_status))
+      continue;
+
+    // A confidence level >= 10 means that the encoding is expected to properly
+    // decode the text. Drop all encodings with lower confidence level.
+    if (confidence < 10)
+      continue;
+
     encodings->push_back(encoding_name);
+    sniffed_encodings.insert(encoding_name);
+  }
+
+  // Append all encodings not included earlier, in arbitrary order.
+  // TODO(jshin): This shouldn't be necessary, possible ICU bug.
+  // See also http://crbug.com/65917.
+  UEnumeration* detectable_encodings = ucsdet_getAllDetectableCharsets(detector,
+                                                                       &status);
+  int detectable_count = uenum_count(detectable_encodings, &status);
+  for (int i = 0; i < detectable_count; i++) {
+    int name_length;
+    const char* name_raw = uenum_next(detectable_encodings,
+                                      &name_length,
+                                      &status);
+    std::string name(name_raw, name_length);
+    if (sniffed_encodings.find(name) == sniffed_encodings.end())
+      encodings->push_back(name);
   }
+  uenum_close(detectable_encodings);
 
   ucsdet_close(detector);
   return !encodings->empty();