Reland: Move PVer4 related code from util.* to v4_protocol_manager_util.*

components/safe_browsing_db/v4_protocol_manager_util.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/safe_browsing_db/v4_protocol_manager_util.h"
 
 #include "base/base64.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/rand_util.h"
+#include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
+#include "crypto/sha2.h"
 #include "net/base/escape.h"
 #include "net/http/http_request_headers.h"
+#include "url/url_util.h"
 
 using base::Time;
 using base::TimeDelta;
 
 namespace safe_browsing {
 
+namespace {
+
+std::string Unescape(const std::string& url) {
+  std::string unescaped_str(url);
+  const int kMaxLoopIterations = 1024;
+  size_t old_size = 0;
+  int loop_var = 0;
+  do {
+    old_size = unescaped_str.size();
+    unescaped_str = net::UnescapeURLComponent(
+        unescaped_str,
+        net::UnescapeRule::SPOOFING_AND_CONTROL_CHARS |
+            net::UnescapeRule::SPACES | net::UnescapeRule::PATH_SEPARATORS |
+            net::UnescapeRule::URL_SPECIAL_CHARS_EXCEPT_PATH_SEPARATORS);
+  } while (old_size != unescaped_str.size() &&
+           ++loop_var <= kMaxLoopIterations);
+
+  return unescaped_str;
+}
+
+std::string Escape(const std::string& url) {
+  std::string escaped_str;
+  // The escaped string is larger so allocate double the length to reduce the
+  // chance of the string being grown.
+  escaped_str.reserve(url.length() * 2);
+  const char* kHexString = "0123456789ABCDEF";
+  for (size_t i = 0; i < url.length(); i++) {
+    unsigned char c = static_cast<unsigned char>(url[i]);
+    if (c <= ' ' || c > '~' || c == '#' || c == '%') {
+      escaped_str += '%';
+      escaped_str += kHexString[c >> 4];
+      escaped_str += kHexString[c & 0xf];
+    } else {
+      escaped_str += c;
+    }
+  }
+
+  return escaped_str;
+}
+
+}  // namespace
+
 std::ostream& operator<<(std::ostream& os, const UpdateListIdentifier& id) {
   os << "{hash: " << id.hash() << "; platform_type: " << id.platform_type
      << "; threat_entry_type: " << id.threat_entry_type
      << "; threat_type: " << id.threat_type << "}";
   return os;
 }
 
+#if defined(OS_WIN)
+#define PLATFORM_TYPE WINDOWS_PLATFORM
+#elif defined(OS_LINUX)
+#define PLATFORM_TYPE LINUX_PLATFORM
+#elif defined(OS_MACOSX)
+#define PLATFORM_TYPE OSX_PLATFORM
+#else
+// This should ideally never compile but it is getting compiled on Android.
+// See: https://bugs.chromium.org/p/chromium/issues/detail?id=621647
+// TODO(vakh): Once that bug is fixed, this should be removed. If we leave
+// the platform_type empty, the server won't recognize the request and
+// return an error response which will pollute our UMA metrics.
+#define PLATFORM_TYPE LINUX_PLATFORM
+#endif
+
+const UpdateListIdentifier GetUrlMalwareId() {
+  return UpdateListIdentifier(PLATFORM_TYPE, URL, MALWARE_THREAT);
+}
+
+const UpdateListIdentifier GetUrlSocEngId() {
+  return UpdateListIdentifier(PLATFORM_TYPE, URL, SOCIAL_ENGINEERING_PUBLIC);
+}
+
 // The Safe Browsing V4 server URL prefix.
 const char kSbV4UrlPrefix[] = "https://safebrowsing.googleapis.com/v4";
 
 bool UpdateListIdentifier::operator==(const UpdateListIdentifier& other) const {
   return platform_type == other.platform_type &&
          threat_entry_type == other.threat_entry_type &&
          threat_type == other.threat_type;
 }
 
 bool UpdateListIdentifier::operator!=(const UpdateListIdentifier& other) const {
@@ skipping 95 matching lines @@
 }
 
 // static
 void V4ProtocolManagerUtil::UpdateHeaders(net::HttpRequestHeaders* headers) {
   // NOTE(vakh): The following header informs the envelope server (which sits in
   // front of Google's stubby server) that the received GET request should be
   // interpreted as a POST.
   headers->SetHeaderIfMissing("X-HTTP-Method-Override", "POST");
 }
 
+// static
+void V4ProtocolManagerUtil::UrlToFullHashes(
+    const GURL& url,
+    base::hash_set<FullHash>* full_hashes) {
+  std::string canon_host, canon_path, canon_query;
+  CanonicalizeUrl(url, &canon_host, &canon_path, &canon_query);
+
+  std::vector<std::string> hosts;
+  if (url.HostIsIPAddress()) {
+    hosts.push_back(url.host());
+  } else {
+    GenerateHostVariantsToCheck(canon_host, &hosts);
+  }
+
+  std::vector<std::string> paths;
+  GeneratePathVariantsToCheck(canon_path, canon_query, &paths);
+  for (const std::string& host : hosts) {
+    for (const std::string& path : paths) {
+      full_hashes->insert(crypto::SHA256HashString(host + path));
+    }
+  }
+}
+
+// static
+void V4ProtocolManagerUtil::GenerateHostsToCheck(
+    const GURL& url,
+    std::vector<std::string>* hosts) {
+  std::string canon_host;
+  CanonicalizeUrl(url, &canon_host, NULL, NULL);
+  GenerateHostVariantsToCheck(canon_host, hosts);
+}
+
+// static
+void V4ProtocolManagerUtil::GeneratePathsToCheck(
+    const GURL& url,
+    std::vector<std::string>* paths) {
+  std::string canon_path;
+  std::string canon_query;
+  CanonicalizeUrl(url, NULL, &canon_path, &canon_query);
+  GeneratePathVariantsToCheck(canon_path, canon_query, paths);
+}
+
+// static
+void V4ProtocolManagerUtil::GeneratePatternsToCheck(
+    const GURL& url,
+    std::vector<std::string>* urls) {
+  std::string canon_host;
+  std::string canon_path;
+  std::string canon_query;
+  CanonicalizeUrl(url, &canon_host, &canon_path, &canon_query);
+
+  std::vector<std::string> hosts, paths;
+  GenerateHostVariantsToCheck(canon_host, &hosts);
+  GeneratePathVariantsToCheck(canon_path, canon_query, &paths);
+  for (size_t h = 0; h < hosts.size(); ++h) {
+    for (size_t p = 0; p < paths.size(); ++p) {
+      urls->push_back(hosts[h] + paths[p]);
+    }
+  }
+}
+
+// static
+void V4ProtocolManagerUtil::CanonicalizeUrl(const GURL& url,
+                                            std::string* canonicalized_hostname,
+                                            std::string* canonicalized_path,
+                                            std::string* canonicalized_query) {
+  DCHECK(url.is_valid());
+
+  // We only canonicalize "normal" URLs.
+  if (!url.IsStandard())
+    return;
+
+  // Following canonicalization steps are excluded since url parsing takes care
+  // of those :-
+  // 1. Remove any tab (0x09), CR (0x0d), and LF (0x0a) chars from url.
+  //    (Exclude escaped version of these chars).
+  // 2. Normalize hostname to 4 dot-seperated decimal values.
+  // 3. Lowercase hostname.
+  // 4. Resolve path sequences "/../" and "/./".
+
+  // That leaves us with the following :-
+  // 1. Remove fragment in URL.
+  GURL url_without_fragment;
+  GURL::Replacements f_replacements;
+  f_replacements.ClearRef();
+  f_replacements.ClearUsername();
+  f_replacements.ClearPassword();
+  url_without_fragment = url.ReplaceComponents(f_replacements);
+
+  // 2. Do URL unescaping until no more hex encoded characters exist.
+  std::string url_unescaped_str(Unescape(url_without_fragment.spec()));
+  url::Parsed parsed;
+  url::ParseStandardURL(url_unescaped_str.data(), url_unescaped_str.length(),
+                        &parsed);
+
+  // 3. In hostname, remove all leading and trailing dots.
+  base::StringPiece host;
+  if (parsed.host.len > 0)
+    host.set(url_unescaped_str.data() + parsed.host.begin, parsed.host.len);
+
+  base::StringPiece host_without_end_dots =
+      base::TrimString(host, ".", base::TrimPositions::TRIM_ALL);
+
+  // 4. In hostname, replace consecutive dots with a single dot.
+  std::string host_without_consecutive_dots(
+      RemoveConsecutiveChars(host_without_end_dots, '.'));
+
+  // 5. In path, replace runs of consecutive slashes with a single slash.
+  base::StringPiece path;
+  if (parsed.path.len > 0)
+    path.set(url_unescaped_str.data() + parsed.path.begin, parsed.path.len);
+  std::string path_without_consecutive_slash(RemoveConsecutiveChars(path, '/'));
+
+  url::Replacements<char> hp_replacements;
+  hp_replacements.SetHost(
+      host_without_consecutive_dots.data(),
+      url::Component(0, host_without_consecutive_dots.length()));
+  hp_replacements.SetPath(
+      path_without_consecutive_slash.data(),
+      url::Component(0, path_without_consecutive_slash.length()));
+
+  std::string url_unescaped_with_can_hostpath;
+  url::StdStringCanonOutput output(&url_unescaped_with_can_hostpath);
+  url::Parsed temp_parsed;
+  url::ReplaceComponents(url_unescaped_str.data(), url_unescaped_str.length(),
+                         parsed, hp_replacements, NULL, &output, &temp_parsed);
+  output.Complete();
+
+  // 6. Step needed to revert escaping done in url::ReplaceComponents.
+  url_unescaped_with_can_hostpath = Unescape(url_unescaped_with_can_hostpath);
+
+  // 7. After performing all above steps, percent-escape all chars in url which
+  // are <= ASCII 32, >= 127, #, %. Escapes must be uppercase hex characters.
+  std::string escaped_canon_url_str(Escape(url_unescaped_with_can_hostpath));
+  url::Parsed final_parsed;
+  url::ParseStandardURL(escaped_canon_url_str.data(),
+                        escaped_canon_url_str.length(), &final_parsed);
+
+  if (canonicalized_hostname && final_parsed.host.len > 0) {
+    *canonicalized_hostname = escaped_canon_url_str.substr(
+        final_parsed.host.begin, final_parsed.host.len);
+  }
+  if (canonicalized_path && final_parsed.path.len > 0) {
+    *canonicalized_path = escaped_canon_url_str.substr(final_parsed.path.begin,
+                                                       final_parsed.path.len);
+  }
+  if (canonicalized_query && final_parsed.query.len > 0) {
+    *canonicalized_query = escaped_canon_url_str.substr(
+        final_parsed.query.begin, final_parsed.query.len);
+  }
+}
+
+// static
+std::string V4ProtocolManagerUtil::RemoveConsecutiveChars(base::StringPiece str,
+                                                          const char c) {
+  std::string output;
+  // Output is at most the length of the original string.
+  output.reserve(str.size());
+
+  size_t i = 0;
+  while (i < str.size()) {
+    output.append(1, str[i++]);
+    if (str[i - 1] == c) {
+      while (i < str.size() && str[i] == c) {
+        i++;
+      }
+    }
+  }
+
+  return output;
+}
+
+// static
+void V4ProtocolManagerUtil::GenerateHostVariantsToCheck(
+    const std::string& host,
+    std::vector<std::string>* hosts) {
+  hosts->clear();
+
+  if (host.empty())
+    return;
+
+  // Per the Safe Browsing Protocol v2 spec, we try the host, and also up to 4
+  // hostnames formed by starting with the last 5 components and successively
+  // removing the leading component.  The last component isn't examined alone,
+  // since it's the TLD or a subcomponent thereof.
+  //
+  // Note that we don't need to be clever about stopping at the "real" eTLD --
+  // the data on the server side has been filtered to ensure it will not
+  // blacklist a whole TLD, and it's not significantly slower on our side to
+  // just check too much.
+  //
+  // Also note that because we have a simple blacklist, not some sort of complex
+  // whitelist-in-blacklist or vice versa, it doesn't matter what order we check
+  // these in.
+  const size_t kMaxHostsToCheck = 4;
+  bool skipped_last_component = false;
+  for (std::string::const_reverse_iterator i(host.rbegin());
+       i != host.rend() && hosts->size() < kMaxHostsToCheck; ++i) {
+    if (*i == '.') {
+      if (skipped_last_component)
+        hosts->push_back(std::string(i.base(), host.end()));
+      else
+        skipped_last_component = true;
+    }
+  }
+  hosts->push_back(host);
+}
+
+// static
+void V4ProtocolManagerUtil::GeneratePathVariantsToCheck(
+    const std::string& path,
+    const std::string& query,
+    std::vector<std::string>* paths) {
+  paths->clear();
+
+  if (path.empty())
+    return;
+
+  // Per the Safe Browsing Protocol v2 spec, we try the exact path with/without
+  // the query parameters, and also up to 4 paths formed by starting at the root
+  // and adding more path components.
+  //
+  // As with the hosts above, it doesn't matter what order we check these in.
+  const size_t kMaxPathsToCheck = 4;
+  for (std::string::const_iterator i(path.begin());
+       i != path.end() && paths->size() < kMaxPathsToCheck; ++i) {
+    if (*i == '/')
+      paths->push_back(std::string(path.begin(), i + 1));
+  }
+
+  if (!paths->empty() && paths->back() != path)
+    paths->push_back(path);
+
+  if (!query.empty())
+    paths->push_back(path + "?" + query);
+}
+
 }  // namespace safe_browsing