FTP: Detect the character encoding only after the entire listing is received.

net/ftp/ftp_directory_listing_parser.cc

Index: net/ftp/ftp_directory_listing_parser.cc
diff --git a/net/ftp/ftp_directory_listing_parser.cc b/net/ftp/ftp_directory_listing_parser.cc
index 2f3eeca1efc9332b178cd26cec3a674920b8bb68..8aa3696073814c8ad872a37f9c1ef05bf3935106 100644
--- a/net/ftp/ftp_directory_listing_parser.cc
+++ b/net/ftp/ftp_directory_listing_parser.cc
@@ -1,12 +1,105 @@
-// Copyright (c) 2009 The Chromium Authors. All rights reserved.  Use of this
-// source code is governed by a BSD-style license that can be found in the
-// LICENSE file.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
 
 #include "net/ftp/ftp_directory_listing_parser.h"
 
+#include "base/i18n/icu_encoding_detection.h"
+#include "base/i18n/icu_string_conversions.h"
+#include "base/stl_util-inl.h"
+#include "base/string_split.h"
+#include "base/string_util.h"
+#include "net/base/net_errors.h"
+#include "net/ftp/ftp_directory_listing_parser_ls.h"
+#include "net/ftp/ftp_directory_listing_parser_netware.h"
+#include "net/ftp/ftp_directory_listing_parser_vms.h"
+#include "net/ftp/ftp_directory_listing_parser_windows.h"
+#include "net/ftp/ftp_server_type_histograms.h"
+
+namespace {
+
+// Converts a string with unknown character encoding to UTF-16. On success
+// fills in |converted_text| and |encoding|. Returns network error code.
+int ConvertStringToUTF16(const std::string& text,
+                         string16* converted_text,
+                         std::string* encoding) {
+  std::vector<std::string> encodings;
+  if (!base::DetectAllEncodings(text, &encodings))
+    return net::ERR_ENCODING_DETECTION_FAILED;
+
+  // Use first encoding that can be used to decode the text.

[eroman, 2011/03/24 23:09:35]

I wander if it would be more flexible to move this loop further out.

In other words, rather than using the first encoding the decodes the text, use
the first encoding whose decoded text is parseable:

for each possible encoding:
  decode to UTF16
  split into lines
  for each directory listing type:
    try to parse all lines
    if successful, return OK (else continue trying parsers/decoders)

[Paweł Hajdan Jr., 2011/03/26 09:47:50]

Yes, it sounds like a good idea. I'd prefer to do it in a separate patch though,
to avoid doing too much in one CL (this is already quite big).

+  for (size_t i = 0; i < encodings.size(); i++) {
+    if (base::CodepageToUTF16(text,
+                              encodings[i].c_str(),
+                              base::OnStringConversionError::FAIL,
+                              converted_text)) {
+      *encoding = encodings[i];
+      return net::OK;
+    }
+  }
+
+  return net::ERR_ENCODING_DETECTION_FAILED;
+}
+
+int FillInRawName(const std::string& encoding,
+                  std::vector<net::FtpDirectoryListingEntry>* entries) {
+  for (size_t i = 0; i < entries->size(); i++) {
+    if (!base::UTF16ToCodepage(entries->at(i).name, encoding.c_str(),
+                               base::OnStringConversionError::FAIL,
+                               &entries->at(i).raw_name)) {
+      return net::ERR_ENCODING_CONVERSION_FAILED;
+    }
+  }
+
+  return net::OK;
+}
+
+}  // namespace
+
 namespace net {
 
-FtpDirectoryListingParser::~FtpDirectoryListingParser() {
+int ParseFtpDirectoryListing(const std::string& text,
+                             const base::Time& current_time,
+                             std::vector<FtpDirectoryListingEntry>* entries) {
+  std::string encoding;
+
+  string16 converted_text;
+  int rv = ConvertStringToUTF16(text, &converted_text, &encoding);
+  if (rv != OK)
+    return rv;
+
+  std::vector<string16> lines;
+  base::SplitString(converted_text, '\n', &lines);
+
+  // TODO(phajdan.jr): Use a table of callbacks instead of repeating code.
+
+  entries->clear();
+  if (ParseFtpDirectoryListingLs(lines, current_time, entries)) {
+    UpdateFtpServerTypeHistograms(SERVER_LS);
+    return FillInRawName(encoding, entries);
+  }
+
+  entries->clear();
+  if (ParseFtpDirectoryListingWindows(lines, entries)) {
+    UpdateFtpServerTypeHistograms(SERVER_WINDOWS);
+    return FillInRawName(encoding, entries);
+  }
+
+  entries->clear();
+  if (ParseFtpDirectoryListingVms(lines, entries)) {
+    UpdateFtpServerTypeHistograms(SERVER_VMS);
+    return FillInRawName(encoding, entries);
+  }
+
+  entries->clear();
+  if (ParseFtpDirectoryListingNetware(lines, current_time, entries)) {
+    UpdateFtpServerTypeHistograms(SERVER_NETWARE);
+    return FillInRawName(encoding, entries);
+  }
+
+  entries->clear();
+  UpdateFtpServerTypeHistograms(SERVER_UNKNOWN);
+  return ERR_UNRECOGNIZED_FTP_DIRECTORY_LISTING_FORMAT;
 }
 
 }  // namespace