FTP: improve character encoding detection in cases where ICU's first guess is wrong.

net/ftp/ftp_directory_listing_buffer.h

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef NET_FTP_FTP_DIRECTORY_LISTING_BUFFER_H_
 #define NET_FTP_FTP_DIRECTORY_LISTING_BUFFER_H_
 #pragma once
 
 #include <deque>
 #include <set>
@@ skipping 33 matching lines @@
 
   // Returns recognized server type. It is valid to call this function at any
   // time, although it will return SERVER_UNKNOWN if it doesn't know the answer.
   FtpServerType GetServerType() const;
 
   const std::string& encoding() const { return encoding_; }
 
  private:
   typedef std::set<FtpDirectoryListingParser*> ParserSet;
 
-  // Converts the string |from| to detected encoding and stores it in |to|.
-  // Returns true on success.
-  bool ConvertToDetectedEncoding(const std::string& from, string16* to);
+  // Decodes the raw buffer using specified |encoding|. On success
+  // clears the raw buffer and appends data to |converted_buffer_|.
+  // Returns network error code.
+  int DecodeBufferUsingEncoding(const std::string& encoding);
 
-  // Tries to extract full lines from the raw buffer, converting them to the
-  // detected encoding. Returns network error code.
-  int ExtractFullLinesFromBuffer();
+  // Converts the raw buffer to UTF-16. Returns network error code.
+  int ConvertBufferToUTF16();
+
+  // Extracts lines from the converted buffer, and puts them in |lines_|.
+  void ExtractFullLinesFromBuffer();
+
+  // Consumes the raw buffer (i.e. does the character set conversion
+  // and line splitting). Returns network error code.
+  int ConsumeBuffer();
 
   // Tries to parse full lines stored in |lines_|. Returns network error code.
   int ParseLines();
 
   // Called when we received the entire input. Propagates that info to remaining
   // parsers. Returns network error code.
   int OnEndOfInput();
 
-  // Detected encoding of the response (empty if unknown or ASCII).
+  // Detected encoding of the response (empty if unknown).
   std::string encoding_;
 
-  // Buffer to keep not-yet-split data.
+  // Buffer to keep data before character set conversion.
   std::string buffer_;
 
+  // Buffer to keep data before line splitting.
+  string16 converted_buffer_;
+
   // CRLF-delimited lines, without the CRLF, not yet consumed by parser.
   std::deque<string16> lines_;
 
   // A collection of parsers for different listing styles. The parsers are owned
   // by this FtpDirectoryListingBuffer.
   ParserSet parsers_;
 
   // When we're sure about the listing format, its parser is stored in
   // |current_parser_|.
   FtpDirectoryListingParser* current_parser_;
 
   DISALLOW_COPY_AND_ASSIGN(FtpDirectoryListingBuffer);
 };
 
 }  // namespace net
 
 #endif  // NET_FTP_FTP_DIRECTORY_LISTING_BUFFER_H_