FTP: improve character encoding detection in cases where ICU's first guess is wrong.

net/ftp/ftp_directory_listing_buffer.cc

 // Copyright (c) 2009 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/ftp/ftp_directory_listing_buffer.h"
 
 #include "base/i18n/icu_encoding_detection.h"
 #include "base/i18n/icu_string_conversions.h"
 #include "base/stl_util-inl.h"
 #include "base/string_util.h"
@@ skipping 19 matching lines @@
 }
 
 FtpDirectoryListingBuffer::~FtpDirectoryListingBuffer() {
   STLDeleteElements(&parsers_);
 }
 
 int FtpDirectoryListingBuffer::ConsumeData(const char* data, int data_length) {
   buffer_.append(data, data_length);
 
   if (!encoding_.empty() || buffer_.length() > 1024) {
-    int rv = ExtractFullLinesFromBuffer();
+    int rv = ConsumeBuffer();
     if (rv != OK)
       return rv;
   }
 
   return ParseLines();
 }
 
 int FtpDirectoryListingBuffer::ProcessRemainingData() {
-  int rv = ExtractFullLinesFromBuffer();
+  int rv = ConsumeBuffer();
   if (rv != OK)
     return rv;
 
-  if (!buffer_.empty())
+  DCHECK(buffer_.empty());
+  if (!converted_buffer_.empty())
     return ERR_INVALID_RESPONSE;
 
   rv = ParseLines();
   if (rv != OK)
     return rv;
 
   rv = OnEndOfInput();
   if (rv != OK)
     return rv;
 
   return OK;
 }
 
 bool FtpDirectoryListingBuffer::EntryAvailable() const {
   return (current_parser_ ? current_parser_->EntryAvailable() : false);
 }
 
 FtpDirectoryListingEntry FtpDirectoryListingBuffer::PopEntry() {
   DCHECK(EntryAvailable());
   return current_parser_->PopEntry();
 }
 
 FtpServerType FtpDirectoryListingBuffer::GetServerType() const {
   return (current_parser_ ? current_parser_->GetServerType() : SERVER_UNKNOWN);
 }
 
-bool FtpDirectoryListingBuffer::ConvertToDetectedEncoding(
-    const std::string& from, string16* to) {
-  std::string encoding(encoding_.empty() ? "ascii" : encoding_);
-  return base::CodepageToUTF16(from, encoding.c_str(),
-                               base::OnStringConversionError::FAIL, to);
-}
+int FtpDirectoryListingBuffer::ConvertBufferToUTF16() {
+  if (encoding_.empty()) {
+    std::vector<std::string> encodings;
+    base::DetectAllEncodings(buffer_, &encodings);
 
-int FtpDirectoryListingBuffer::ExtractFullLinesFromBuffer() {
-  if (encoding_.empty()) {
-    if (!base::DetectEncoding(buffer_, &encoding_))
+    // Use first encoding that can be used to decode the buffer.
+    for (size_t i = 0; i < encodings.size(); i++) {
+      string16 converted;
+      if (base::CodepageToUTF16(buffer_,
+                                encodings[i].c_str(),
+                                base::OnStringConversionError::FAIL,
+                                &converted)) {
+        encoding_ = encodings[i];
+        break;
+      }
+    }
+
+    // If we haven't detected the encoding at this point, it's an error.
+    if (encoding_.empty())
       return ERR_ENCODING_DETECTION_FAILED;

[eroman, 2010/11/16 00:35:03]

It seems prudent to clear converted_buffer_ before we return failures.

[Paweł Hajdan Jr., 2010/11/16 21:22:39]

Skipped, as discussed offline.

   }
 
+  string16 converted;
+  if (!base::CodepageToUTF16(buffer_,

[eroman, 2010/11/16 00:35:03]

The structure of this function seems a little redundant. We already may have
converted successfully to UTF16 above, however we throw that work away and
re-convert down here. Can this be re-ordered a bit?

[Paweł Hajdan Jr., 2010/11/16 21:22:39]

Done.

+                             encoding_.c_str(),
+                             base::OnStringConversionError::FAIL,
+                             &converted))
+    return ERR_ENCODING_CONVERSION_FAILED;
+
+  buffer_.clear();
+  converted_buffer_ += converted;
+  return OK;
+}
+
+void FtpDirectoryListingBuffer::ExtractFullLinesFromBuffer() {
   int cut_pos = 0;
   // TODO(phajdan.jr): This code accepts all endlines matching \r*\n. Should it
   // be more strict, or enforce consistent line endings?
-  for (size_t i = 0; i < buffer_.length(); ++i) {
-    if (buffer_[i] != '\n')
+  for (size_t i = 0; i < converted_buffer_.length(); ++i) {
+    if (converted_buffer_[i] != '\n')
       continue;
     int line_length = i - cut_pos;
-    if (i >= 1 && buffer_[i - 1] == '\r')
+    if (i >= 1 && converted_buffer_[i - 1] == '\r')
       line_length--;
-    std::string line(buffer_.substr(cut_pos, line_length));
+    lines_.push_back(converted_buffer_.substr(cut_pos, line_length));
     cut_pos = i + 1;
-    string16 line_converted;
-    if (!ConvertToDetectedEncoding(line, &line_converted)) {
-      buffer_.erase(0, cut_pos);
-      return ERR_ENCODING_CONVERSION_FAILED;
-    }
-    lines_.push_back(line_converted);
   }
-  buffer_.erase(0, cut_pos);
+  converted_buffer_.erase(0, cut_pos);
+}
+
+int FtpDirectoryListingBuffer::ConsumeBuffer() {
+  int rv = ConvertBufferToUTF16();
+  if (rv != OK)
+    return rv;
+
+  ExtractFullLinesFromBuffer();
   return OK;
 }
 
 int FtpDirectoryListingBuffer::ParseLines() {
   while (!lines_.empty()) {
     string16 line = lines_.front();
     lines_.pop_front();
     if (current_parser_) {
       if (!current_parser_->ConsumeLine(line))
         return ERR_FAILED;
@@ skipping 42 matching lines @@
       return OK;
 
     return ERR_UNRECOGNIZED_FTP_DIRECTORY_LISTING_FORMAT;
   }
 
   current_parser_ = *parsers_.begin();
   return OK;
 }
 
 }  // namespace net