Revert 262487 "Add net/base/filename_util.h."

trunk/src/net/base/net_util.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/base/net_util.h"
 
-#include <errno.h>
-
 #include <algorithm>
 #include <iterator>
 #include <map>
-#include <set>
 
 #include "build/build_config.h"
 
 #if defined(OS_WIN)
 #include <windows.h>
 #include <iphlpapi.h>
 #include <winsock2.h>
 #pragma comment(lib, "iphlpapi.lib")
 #elif defined(OS_POSIX)
 #include <fcntl.h>
 #if !defined(OS_ANDROID)
 #include <ifaddrs.h>
 #endif
 #include <net/if.h>
 #include <netdb.h>
 #include <netinet/in.h>
 #endif
 
 #include "base/basictypes.h"
+#include "base/file_util.h"
+#include "base/files/file_path.h"
+#include "base/i18n/file_util_icu.h"
+#include "base/i18n/icu_string_conversions.h"
 #include "base/i18n/time_formatting.h"
 #include "base/json/string_escape.h"
 #include "base/lazy_instance.h"
 #include "base/logging.h"
 #include "base/memory/singleton.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
+#include "base/path_service.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_piece.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_tokenizer.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/strings/sys_string_conversions.h"
 #include "base/strings/utf_offset_string_conversions.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/synchronization/lock.h"
 #include "base/sys_byteorder.h"
 #include "base/time/time.h"
 #include "base/values.h"
 #include "grit/net_resources.h"
 #include "url/gurl.h"
 #include "url/url_canon.h"
 #include "url/url_canon_ip.h"
 #include "url/url_parse.h"
 #if defined(OS_ANDROID)
 #include "net/android/network_library.h"
 #endif
 #include "net/base/dns_util.h"
 #include "net/base/escape.h"
+#include "net/base/mime_util.h"
 #include "net/base/net_module.h"
 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
 #if defined(OS_WIN)
 #include "net/base/winsock_init.h"
 #endif
 #include "net/http/http_content_disposition.h"
 #include "third_party/icu/source/common/unicode/uidna.h"
 #include "third_party/icu/source/common/unicode/uniset.h"
 #include "third_party/icu/source/common/unicode/uscript.h"
 #include "third_party/icu/source/common/unicode/uset.h"
 #include "third_party/icu/source/i18n/unicode/datefmt.h"
 #include "third_party/icu/source/i18n/unicode/regex.h"
 #include "third_party/icu/source/i18n/unicode/ulocdata.h"
 
 using base::Time;
 
 namespace net {
 
 namespace {
 
 typedef std::vector<size_t> Offsets;
 
+// what we prepend to get a file URL
+static const base::FilePath::CharType kFileURLPrefix[] =
+    FILE_PATH_LITERAL("file:///");
+
 // The general list of blocked ports. Will be blocked unless a specific
 // protocol overrides it. (Ex: ftp can use ports 20 and 21)
 static const int kRestrictedPorts[] = {
   1,    // tcpmux
   7,    // echo
   9,    // discard
   11,   // systat
   13,   // daytime
   15,   // netstat
   17,   // qotd
@@ skipping 647 matching lines @@
     if (output_component) {
       output_component->begin = static_cast<int>(output_component_begin);
       output_component->len =
           static_cast<int>(output->length() - output_component_begin);
     }
   } else if (output_component) {
     output_component->reset();
   }
 }
 
+void SanitizeGeneratedFileName(base::FilePath::StringType* filename,
+                               bool replace_trailing) {
+  const base::FilePath::CharType kReplace[] = FILE_PATH_LITERAL("-");
+  if (filename->empty())
+    return;
+  if (replace_trailing) {
+    // Handle CreateFile() stripping trailing dots and spaces on filenames
+    // http://support.microsoft.com/kb/115827
+    size_t length = filename->size();
+    size_t pos = filename->find_last_not_of(FILE_PATH_LITERAL(" ."));
+    filename->resize((pos == std::string::npos) ? 0 : (pos + 1));
+    base::TrimWhitespace(*filename, base::TRIM_TRAILING, filename);
+    if (filename->empty())
+      return;
+    size_t trimmed = length - filename->size();
+    if (trimmed)
+      filename->insert(filename->end(), trimmed, kReplace[0]);
+  }
+  base::TrimString(*filename, FILE_PATH_LITERAL("."), filename);
+  if (filename->empty())
+    return;
+  // Replace any path information by changing path separators.
+  ReplaceSubstringsAfterOffset(filename, 0, FILE_PATH_LITERAL("/"), kReplace);
+  ReplaceSubstringsAfterOffset(filename, 0, FILE_PATH_LITERAL("\\"), kReplace);
+}
+
+// Returns the filename determined from the last component of the path portion
+// of the URL.  Returns an empty string if the URL doesn't have a path or is
+// invalid. If the generated filename is not reliable,
+// |should_overwrite_extension| will be set to true, in which case a better
+// extension should be determined based on the content type.
+std::string GetFileNameFromURL(const GURL& url,
+                               const std::string& referrer_charset,
+                               bool* should_overwrite_extension) {
+  // about: and data: URLs don't have file names, but esp. data: URLs may
+  // contain parts that look like ones (i.e., contain a slash).  Therefore we
+  // don't attempt to divine a file name out of them.
+  if (!url.is_valid() || url.SchemeIs("about") || url.SchemeIs("data"))
+    return std::string();
+
+  const std::string unescaped_url_filename = UnescapeURLComponent(
+      url.ExtractFileName(),
+      UnescapeRule::SPACES | UnescapeRule::URL_SPECIAL_CHARS);
+
+  // The URL's path should be escaped UTF-8, but may not be.
+  std::string decoded_filename = unescaped_url_filename;
+  if (!IsStringUTF8(decoded_filename)) {
+    // TODO(jshin): this is probably not robust enough. To be sure, we need
+    // encoding detection.
+    base::string16 utf16_output;
+    if (!referrer_charset.empty() &&
+        base::CodepageToUTF16(unescaped_url_filename,
+                              referrer_charset.c_str(),
+                              base::OnStringConversionError::FAIL,
+                              &utf16_output)) {
+      decoded_filename = base::UTF16ToUTF8(utf16_output);
+    } else {
+      decoded_filename = base::WideToUTF8(
+          base::SysNativeMBToWide(unescaped_url_filename));
+    }
+  }
+  // If the URL contains a (possibly empty) query, assume it is a generator, and
+  // allow the determined extension to be overwritten.
+  *should_overwrite_extension = !decoded_filename.empty() && url.has_query();
+
+  return decoded_filename;
+}
+
+// Returns whether the specified extension is automatically integrated into the
+// windows shell.
+bool IsShellIntegratedExtension(const base::FilePath::StringType& extension) {
+  base::FilePath::StringType extension_lower = StringToLowerASCII(extension);
+
+  // http://msdn.microsoft.com/en-us/library/ms811694.aspx
+  // Right-clicking on shortcuts can be magical.
+  if ((extension_lower == FILE_PATH_LITERAL("local")) ||
+      (extension_lower == FILE_PATH_LITERAL("lnk")))
+    return true;
+
+  // http://www.juniper.net/security/auto/vulnerabilities/vuln2612.html
+  // Files become magical if they end in a CLSID, so block such extensions.
+  if (!extension_lower.empty() &&
+      (extension_lower[0] == FILE_PATH_LITERAL('{')) &&
+      (extension_lower[extension_lower.length() - 1] == FILE_PATH_LITERAL('}')))
+    return true;
+  return false;
+}
+
+// Returns whether the specified file name is a reserved name on windows.
+// This includes names like "com2.zip" (which correspond to devices) and
+// desktop.ini and thumbs.db which have special meaning to the windows shell.
+bool IsReservedName(const base::FilePath::StringType& filename) {
+  // This list is taken from the MSDN article "Naming a file"
+  // http://msdn2.microsoft.com/en-us/library/aa365247(VS.85).aspx
+  // I also added clock$ because GetSaveFileName seems to consider it as a
+  // reserved name too.
+  static const char* const known_devices[] = {
+    "con", "prn", "aux", "nul", "com1", "com2", "com3", "com4", "com5",
+    "com6", "com7", "com8", "com9", "lpt1", "lpt2", "lpt3", "lpt4",
+    "lpt5", "lpt6", "lpt7", "lpt8", "lpt9", "clock$"
+  };
+#if defined(OS_WIN)
+  std::string filename_lower = StringToLowerASCII(base::WideToUTF8(filename));
+#elif defined(OS_POSIX)
+  std::string filename_lower = StringToLowerASCII(filename);
+#endif
+
+  for (size_t i = 0; i < arraysize(known_devices); ++i) {
+    // Exact match.
+    if (filename_lower == known_devices[i])
+      return true;
+    // Starts with "DEVICE.".
+    if (filename_lower.find(std::string(known_devices[i]) + ".") == 0)
+      return true;
+  }
+
+  static const char* const magic_names[] = {
+    // These file names are used by the "Customize folder" feature of the shell.
+    "desktop.ini",
+    "thumbs.db",
+  };
+
+  for (size_t i = 0; i < arraysize(magic_names); ++i) {
+    if (filename_lower == magic_names[i])
+      return true;
+  }
+
+  return false;
+}
+
+// Examines the current extension in |file_name| and modifies it if necessary in
+// order to ensure the filename is safe.  If |file_name| doesn't contain an
+// extension or if |ignore_extension| is true, then a new extension will be
+// constructed based on the |mime_type|.
+//
+// We're addressing two things here:
+//
+// 1) Usability.  If there is no reliable file extension, we want to guess a
+//    reasonable file extension based on the content type.
+//
+// 2) Shell integration.  Some file extensions automatically integrate with the
+//    shell.  We block these extensions to prevent a malicious web site from
+//    integrating with the user's shell.
+void EnsureSafeExtension(const std::string& mime_type,
+                         bool ignore_extension,
+                         base::FilePath* file_name) {
+  // See if our file name already contains an extension.
+  base::FilePath::StringType extension = file_name->Extension();
+  if (!extension.empty())
+    extension.erase(extension.begin());  // Erase preceding '.'.
+
+  if ((ignore_extension || extension.empty()) && !mime_type.empty()) {
+    base::FilePath::StringType preferred_mime_extension;
+    std::vector<base::FilePath::StringType> all_mime_extensions;
+    // The GetPreferredExtensionForMimeType call will end up going to disk.  Do
+    // this on another thread to avoid slowing the IO thread.
+    // http://crbug.com/61827
+    // TODO(asanka): Remove this ScopedAllowIO once all callers have switched
+    // over to IO safe threads.
+    base::ThreadRestrictions::ScopedAllowIO allow_io;
+    net::GetPreferredExtensionForMimeType(mime_type, &preferred_mime_extension);
+    net::GetExtensionsForMimeType(mime_type, &all_mime_extensions);
+    // If the existing extension is in the list of valid extensions for the
+    // given type, use it. This avoids doing things like pointlessly renaming
+    // "foo.jpg" to "foo.jpeg".
+    if (std::find(all_mime_extensions.begin(),
+                  all_mime_extensions.end(),
+                  extension) != all_mime_extensions.end()) {
+      // leave |extension| alone
+    } else if (!preferred_mime_extension.empty()) {
+      extension = preferred_mime_extension;
+    }
+  }
+
+#if defined(OS_WIN)
+  static const base::FilePath::CharType default_extension[] =
+      FILE_PATH_LITERAL("download");
+
+  // Rename shell-integrated extensions.
+  // TODO(asanka): Consider stripping out the bad extension and replacing it
+  // with the preferred extension for the MIME type if one is available.
+  if (IsShellIntegratedExtension(extension))
+    extension.assign(default_extension);
+#endif
+
+  *file_name = file_name->ReplaceExtension(extension);
+}
+
+bool FilePathToString16(const base::FilePath& path, base::string16* converted) {
+#if defined(OS_WIN)
+  return base::WideToUTF16(
+      path.value().c_str(), path.value().size(), converted);
+#elif defined(OS_POSIX)
+  std::string component8 = path.AsUTF8Unsafe();
+  return !component8.empty() &&
+         base::UTF8ToUTF16(component8.c_str(), component8.size(), converted);
+#endif
+}
+
 bool IPNumberPrefixCheck(const IPAddressNumber& ip_number,
                          const unsigned char* ip_prefix,
                          size_t prefix_length_in_bits) {
   // Compare all the bytes that fall entirely within the prefix.
   int num_entire_bytes_in_prefix = prefix_length_in_bits / 8;
   for (int i = 0; i < num_entire_bytes_in_prefix; ++i) {
     if (ip_number[i] != ip_prefix[i])
       return false;
   }
 
@@ skipping 18 matching lines @@
 const FormatUrlType kFormatUrlOmitAll = kFormatUrlOmitUsernamePassword |
     kFormatUrlOmitHTTP | kFormatUrlOmitTrailingSlashOnBareHostname;
 
 static base::LazyInstance<std::multiset<int> >::Leaky
     g_explicitly_allowed_ports = LAZY_INSTANCE_INITIALIZER;
 
 size_t GetCountOfExplicitlyAllowedPorts() {
   return g_explicitly_allowed_ports.Get().size();
 }
 
+GURL FilePathToFileURL(const base::FilePath& path) {
+  // Produce a URL like "file:///C:/foo" for a regular file, or
+  // "file://///server/path" for UNC. The URL canonicalizer will fix up the
+  // latter case to be the canonical UNC form: "file://server/path"
+  base::FilePath::StringType url_string(kFileURLPrefix);
+  if (!path.IsAbsolute()) {
+    base::FilePath current_dir;
+    PathService::Get(base::DIR_CURRENT, &current_dir);
+    url_string.append(current_dir.value());
+    url_string.push_back(base::FilePath::kSeparators[0]);
+  }
+  url_string.append(path.value());
+
+  // Now do replacement of some characters. Since we assume the input is a
+  // literal filename, anything the URL parser might consider special should
+  // be escaped here.
+
+  // must be the first substitution since others will introduce percents as the
+  // escape character
+  ReplaceSubstringsAfterOffset(&url_string, 0,
+      FILE_PATH_LITERAL("%"), FILE_PATH_LITERAL("%25"));
+
+  // semicolon is supposed to be some kind of separator according to RFC 2396
+  ReplaceSubstringsAfterOffset(&url_string, 0,
+      FILE_PATH_LITERAL(";"), FILE_PATH_LITERAL("%3B"));
+
+  ReplaceSubstringsAfterOffset(&url_string, 0,
+      FILE_PATH_LITERAL("#"), FILE_PATH_LITERAL("%23"));
+
+  ReplaceSubstringsAfterOffset(&url_string, 0,
+      FILE_PATH_LITERAL("?"), FILE_PATH_LITERAL("%3F"));
+
+#if defined(OS_POSIX)
+  ReplaceSubstringsAfterOffset(&url_string, 0,
+      FILE_PATH_LITERAL("\\"), FILE_PATH_LITERAL("%5C"));
+#endif
+
+  return GURL(url_string);
+}
+
 std::string GetSpecificHeader(const std::string& headers,
                               const std::string& name) {
   // We want to grab the Value from the "Key: Value" pairs in the headers,
   // which should look like this (no leading spaces, \n-separated) (we format
   // them this way in url_request_inet.cc):
   //    HTTP/1.1 200 OK\n
   //    ETag: "6d0b8-947-24f35ec0"\n
   //    Content-Length: 2375\n
   //    Content-Type: text/html; charset=UTF-8\n
   //    Last-Modified: Sun, 03 Sep 2006 04:34:43 GMT\n
@@ skipping 148 matching lines @@
 base::string16 StripWWW(const base::string16& text) {
   const base::string16 www(base::ASCIIToUTF16("www."));
   return StartsWith(text, www, true) ? text.substr(www.length()) : text;
 }
 
 base::string16 StripWWWFromHost(const GURL& url) {
   DCHECK(url.is_valid());
   return StripWWW(base::ASCIIToUTF16(url.host()));
 }
 
+bool IsSafePortablePathComponent(const base::FilePath& component) {
+  base::string16 component16;
+  base::FilePath::StringType sanitized = component.value();
+  SanitizeGeneratedFileName(&sanitized, true);
+  base::FilePath::StringType extension = component.Extension();
+  if (!extension.empty())
+    extension.erase(extension.begin());  // Erase preceding '.'.
+  return !component.empty() &&
+         (component == component.BaseName()) &&
+         (component == component.StripTrailingSeparators()) &&
+         FilePathToString16(component, &component16) &&
+         file_util::IsFilenameLegal(component16) &&
+         !IsShellIntegratedExtension(extension) &&
+         (sanitized == component.value()) &&
+         !IsReservedName(component.value());
+}
+
+bool IsSafePortableRelativePath(const base::FilePath& path) {
+  if (path.empty() || path.IsAbsolute() || path.EndsWithSeparator())
+    return false;
+  std::vector<base::FilePath::StringType> components;
+  path.GetComponents(&components);
+  if (components.empty())
+    return false;
+  for (size_t i = 0; i < components.size() - 1; ++i) {
+    if (!IsSafePortablePathComponent(base::FilePath(components[i])))
+      return false;
+  }
+  return IsSafePortablePathComponent(path.BaseName());
+}
+
+void GenerateSafeFileName(const std::string& mime_type,
+                          bool ignore_extension,
+                          base::FilePath* file_path) {
+  // Make sure we get the right file extension
+  EnsureSafeExtension(mime_type, ignore_extension, file_path);
+
+#if defined(OS_WIN)
+  // Prepend "_" to the file name if it's a reserved name
+  base::FilePath::StringType leaf_name = file_path->BaseName().value();
+  DCHECK(!leaf_name.empty());
+  if (IsReservedName(leaf_name)) {
+    leaf_name = base::FilePath::StringType(FILE_PATH_LITERAL("_")) + leaf_name;
+    *file_path = file_path->DirName();
+    if (file_path->value() == base::FilePath::kCurrentDirectory) {
+      *file_path = base::FilePath(leaf_name);
+    } else {
+      *file_path = file_path->Append(leaf_name);
+    }
+  }
+#endif
+}
+
+base::string16 GetSuggestedFilename(const GURL& url,
+                                    const std::string& content_disposition,
+                                    const std::string& referrer_charset,
+                                    const std::string& suggested_name,
+                                    const std::string& mime_type,
+                                    const std::string& default_name) {
+  // TODO: this function to be updated to match the httpbis recommendations.
+  // Talk to abarth for the latest news.
+
+  // We don't translate this fallback string, "download". If localization is
+  // needed, the caller should provide localized fallback in |default_name|.
+  static const base::FilePath::CharType kFinalFallbackName[] =
+    FILE_PATH_LITERAL("download");
+  std::string filename;  // In UTF-8
+  bool overwrite_extension = false;
+
+  // Try to extract a filename from content-disposition first.
+  if (!content_disposition.empty()) {
+    HttpContentDisposition header(content_disposition, referrer_charset);
+    filename = header.filename();
+  }
+
+  // Then try to use the suggested name.
+  if (filename.empty() && !suggested_name.empty())
+    filename = suggested_name;
+
+  // Now try extracting the filename from the URL.  GetFileNameFromURL() only
+  // looks at the last component of the URL and doesn't return the hostname as a
+  // failover.
+  if (filename.empty())
+    filename = GetFileNameFromURL(url, referrer_charset, &overwrite_extension);
+
+  // Finally try the URL hostname, but only if there's no default specified in
+  // |default_name|.  Some schemes (e.g.: file:, about:, data:) do not have a
+  // host name.
+  if (filename.empty() &&
+      default_name.empty() &&
+      url.is_valid() &&
+      !url.host().empty()) {
+    // TODO(jungshik) : Decode a 'punycoded' IDN hostname. (bug 1264451)
+    filename = url.host();
+  }
+
+  bool replace_trailing = false;
+  base::FilePath::StringType result_str, default_name_str;
+#if defined(OS_WIN)
+  replace_trailing = true;
+  result_str = base::UTF8ToUTF16(filename);
+  default_name_str = base::UTF8ToUTF16(default_name);
+#else
+  result_str = filename;
+  default_name_str = default_name;
+#endif
+  SanitizeGeneratedFileName(&result_str, replace_trailing);
+  if (result_str.find_last_not_of(FILE_PATH_LITERAL("-_")) ==
+      base::FilePath::StringType::npos) {
+    result_str = !default_name_str.empty() ? default_name_str :
+      base::FilePath::StringType(kFinalFallbackName);
+    overwrite_extension = false;
+  }
+  file_util::ReplaceIllegalCharactersInPath(&result_str, '-');
+  base::FilePath result(result_str);
+  GenerateSafeFileName(mime_type, overwrite_extension, &result);
+
+  base::string16 result16;
+  if (!FilePathToString16(result, &result16)) {
+    result = base::FilePath(default_name_str);
+    if (!FilePathToString16(result, &result16)) {
+      result = base::FilePath(kFinalFallbackName);
+      FilePathToString16(result, &result16);
+    }
+  }
+  return result16;
+}
+
+base::FilePath GenerateFileName(const GURL& url,
+                                const std::string& content_disposition,
+                                const std::string& referrer_charset,
+                                const std::string& suggested_name,
+                                const std::string& mime_type,
+                                const std::string& default_file_name) {
+  base::string16 file_name = GetSuggestedFilename(url,
+                                                  content_disposition,
+                                                  referrer_charset,
+                                                  suggested_name,
+                                                  mime_type,
+                                                  default_file_name);
+
+#if defined(OS_WIN)
+  base::FilePath generated_name(file_name);
+#else
+  base::FilePath generated_name(
+      base::SysWideToNativeMB(base::UTF16ToWide(file_name)));
+#endif
+
+#if defined(OS_CHROMEOS)
+  // When doing file manager operations on ChromeOS, the file paths get
+  // normalized in WebKit layer, so let's ensure downloaded files have
+  // normalized names. Otherwise, we won't be able to handle files with NFD
+  // utf8 encoded characters in name.
+  file_util::NormalizeFileNameEncoding(&generated_name);
+#endif
+
+  DCHECK(!generated_name.empty());
+
+  return generated_name;
+}
 
 bool IsPortAllowedByDefault(int port) {
   int array_size = arraysize(kRestrictedPorts);
   for (int i = 0; i < array_size; i++) {
     if (kRestrictedPorts[i] == port) {
       return false;
     }
   }
   return true;
 }
@@ skipping 850 matching lines @@
   }
   return a1.size() * CHAR_BIT;
 }
 
 unsigned MaskPrefixLength(const IPAddressNumber& mask) {
   IPAddressNumber all_ones(mask.size(), 0xFF);
   return CommonPrefixLength(mask, all_ones);
 }
 
 }  // namespace net