| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include "components/safe_browsing_db/util.h" | |
| 6 | |
| 7 #include "base/strings/string_util.h" | |
| 8 #include "crypto/sha2.h" | |
| 9 #include "net/base/escape.h" | |
| 10 #include "url/gurl.h" | |
| 11 #include "url/url_util.h" | |
| 12 | |
| 13 // Utility functions ----------------------------------------------------------- | |
| 14 | |
| 15 namespace { | |
| 16 bool IsKnownList(const std::string& name) { | |
| 17 for (size_t i = 0; i < arraysize(safe_browsing::kAllLists); ++i) { | |
| 18 if (!strcmp(safe_browsing::kAllLists[i], name.c_str())) { | |
| 19 return true; | |
| 20 } | |
| 21 } | |
| 22 return false; | |
| 23 } | |
| 24 } // namespace | |
| 25 | |
| 26 | |
| 27 // SBCachedFullHashResult ------------------------------------------------------ | |
| 28 | |
| 29 SBCachedFullHashResult::SBCachedFullHashResult() {} | |
| 30 | |
| 31 SBCachedFullHashResult::SBCachedFullHashResult( | |
| 32 const base::Time& in_expire_after) | |
| 33 : expire_after(in_expire_after) {} | |
| 34 | |
| 35 SBCachedFullHashResult::~SBCachedFullHashResult() {} | |
| 36 | |
| 37 | |
| 38 namespace safe_browsing { | |
| 39 | |
| 40 // Listnames that browser can process. | |
| 41 const char kMalwareList[] = "goog-malware-shavar"; | |
| 42 const char kPhishingList[] = "goog-phish-shavar"; | |
| 43 const char kBinUrlList[] = "goog-badbinurl-shavar"; | |
| 44 const char kCsdWhiteList[] = "goog-csdwhite-sha256"; | |
| 45 const char kDownloadWhiteList[] = "goog-downloadwhite-digest256"; | |
| 46 const char kExtensionBlacklist[] = "goog-badcrxids-digestvar"; | |
| 47 const char kIPBlacklist[] = "goog-badip-digest256"; | |
| 48 const char kUnwantedUrlList[] = "goog-unwanted-shavar"; | |
| 49 const char kInclusionWhitelist[] = "goog-csdinclusionwhite-sha256"; | |
| 50 | |
| 51 const char* kAllLists[9] = { | |
| 52 kMalwareList, | |
| 53 kPhishingList, | |
| 54 kBinUrlList, | |
| 55 kCsdWhiteList, | |
| 56 kDownloadWhiteList, | |
| 57 kExtensionBlacklist, | |
| 58 kIPBlacklist, | |
| 59 kUnwantedUrlList, | |
| 60 kInclusionWhitelist, | |
| 61 }; | |
| 62 | |
| 63 ListType GetListId(const base::StringPiece& name) { | |
| 64 ListType id; | |
| 65 if (name == kMalwareList) { | |
| 66 id = MALWARE; | |
| 67 } else if (name == kPhishingList) { | |
| 68 id = PHISH; | |
| 69 } else if (name == kBinUrlList) { | |
| 70 id = BINURL; | |
| 71 } else if (name == kCsdWhiteList) { | |
| 72 id = CSDWHITELIST; | |
| 73 } else if (name == kDownloadWhiteList) { | |
| 74 id = DOWNLOADWHITELIST; | |
| 75 } else if (name == kExtensionBlacklist) { | |
| 76 id = EXTENSIONBLACKLIST; | |
| 77 } else if (name == kIPBlacklist) { | |
| 78 id = IPBLACKLIST; | |
| 79 } else if (name == kUnwantedUrlList) { | |
| 80 id = UNWANTEDURL; | |
| 81 } else if (name == kInclusionWhitelist) { | |
| 82 id = INCLUSIONWHITELIST; | |
| 83 } else { | |
| 84 id = INVALID; | |
| 85 } | |
| 86 return id; | |
| 87 } | |
| 88 | |
| 89 bool GetListName(ListType list_id, std::string* list) { | |
| 90 switch (list_id) { | |
| 91 case MALWARE: | |
| 92 *list = kMalwareList; | |
| 93 break; | |
| 94 case PHISH: | |
| 95 *list = kPhishingList; | |
| 96 break; | |
| 97 case BINURL: | |
| 98 *list = kBinUrlList; | |
| 99 break; | |
| 100 case CSDWHITELIST: | |
| 101 *list = kCsdWhiteList; | |
| 102 break; | |
| 103 case DOWNLOADWHITELIST: | |
| 104 *list = kDownloadWhiteList; | |
| 105 break; | |
| 106 case EXTENSIONBLACKLIST: | |
| 107 *list = kExtensionBlacklist; | |
| 108 break; | |
| 109 case IPBLACKLIST: | |
| 110 *list = kIPBlacklist; | |
| 111 break; | |
| 112 case UNWANTEDURL: | |
| 113 *list = kUnwantedUrlList; | |
| 114 break; | |
| 115 case INCLUSIONWHITELIST: | |
| 116 *list = kInclusionWhitelist; | |
| 117 break; | |
| 118 default: | |
| 119 return false; | |
| 120 } | |
| 121 DCHECK(IsKnownList(*list)); | |
| 122 return true; | |
| 123 } | |
| 124 | |
| 125 | |
| 126 SBFullHash SBFullHashForString(const base::StringPiece& str) { | |
| 127 SBFullHash h; | |
| 128 crypto::SHA256HashString(str, &h.full_hash, sizeof(h.full_hash)); | |
| 129 return h; | |
| 130 } | |
| 131 | |
| 132 SBFullHash StringToSBFullHash(const std::string& hash_in) { | |
| 133 DCHECK_EQ(crypto::kSHA256Length, hash_in.size()); | |
| 134 SBFullHash hash_out; | |
| 135 memcpy(hash_out.full_hash, hash_in.data(), crypto::kSHA256Length); | |
| 136 return hash_out; | |
| 137 } | |
| 138 | |
| 139 std::string SBFullHashToString(const SBFullHash& hash) { | |
| 140 DCHECK_EQ(crypto::kSHA256Length, sizeof(hash.full_hash)); | |
| 141 return std::string(hash.full_hash, sizeof(hash.full_hash)); | |
| 142 } | |
| 143 | |
| 144 | |
| 145 std::string Unescape(const std::string& url) { | |
| 146 std::string unescaped_str(url); | |
| 147 std::string old_unescaped_str; | |
| 148 const int kMaxLoopIterations = 1024; | |
| 149 int loop_var = 0; | |
| 150 do { | |
| 151 old_unescaped_str = unescaped_str; | |
| 152 unescaped_str = net::UnescapeURLComponent( | |
| 153 old_unescaped_str, net::UnescapeRule::SPOOFING_AND_CONTROL_CHARS | | |
| 154 net::UnescapeRule::SPACES | | |
| 155 net::UnescapeRule::URL_SPECIAL_CHARS); | |
| 156 } while (unescaped_str != old_unescaped_str && ++loop_var <= | |
| 157 kMaxLoopIterations); | |
| 158 | |
| 159 return unescaped_str; | |
| 160 } | |
| 161 | |
| 162 std::string Escape(const std::string& url) { | |
| 163 std::string escaped_str; | |
| 164 const char* kHexString = "0123456789ABCDEF"; | |
| 165 for (size_t i = 0; i < url.length(); i++) { | |
| 166 unsigned char c = static_cast<unsigned char>(url[i]); | |
| 167 if (c <= ' ' || c > '~' || c == '#' || c == '%') { | |
| 168 escaped_str.push_back('%'); | |
| 169 escaped_str.push_back(kHexString[c >> 4]); | |
| 170 escaped_str.push_back(kHexString[c & 0xf]); | |
| 171 } else { | |
| 172 escaped_str.push_back(c); | |
| 173 } | |
| 174 } | |
| 175 | |
| 176 return escaped_str; | |
| 177 } | |
| 178 | |
| 179 std::string RemoveConsecutiveChars(const std::string& str, const char c) { | |
| 180 std::string output(str); | |
| 181 std::string string_to_find; | |
| 182 std::string::size_type loc = 0; | |
| 183 string_to_find.append(2, c); | |
| 184 while ((loc = output.find(string_to_find, loc)) != std::string::npos) { | |
| 185 output.erase(loc, 1); | |
| 186 } | |
| 187 | |
| 188 return output; | |
| 189 } | |
| 190 | |
| 191 // Canonicalizes url as per Google Safe Browsing Specification. | |
| 192 // See section 6.1 in | |
| 193 // http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec. | |
| 194 void CanonicalizeUrl(const GURL& url, | |
| 195 std::string* canonicalized_hostname, | |
| 196 std::string* canonicalized_path, | |
| 197 std::string* canonicalized_query) { | |
| 198 DCHECK(url.is_valid()); | |
| 199 | |
| 200 // We only canonicalize "normal" URLs. | |
| 201 if (!url.IsStandard()) | |
| 202 return; | |
| 203 | |
| 204 // Following canonicalization steps are excluded since url parsing takes care | |
| 205 // of those :- | |
| 206 // 1. Remove any tab (0x09), CR (0x0d), and LF (0x0a) chars from url. | |
| 207 // (Exclude escaped version of these chars). | |
| 208 // 2. Normalize hostname to 4 dot-seperated decimal values. | |
| 209 // 3. Lowercase hostname. | |
| 210 // 4. Resolve path sequences "/../" and "/./". | |
| 211 | |
| 212 // That leaves us with the following :- | |
| 213 // 1. Remove fragment in URL. | |
| 214 GURL url_without_fragment; | |
| 215 GURL::Replacements f_replacements; | |
| 216 f_replacements.ClearRef(); | |
| 217 f_replacements.ClearUsername(); | |
| 218 f_replacements.ClearPassword(); | |
| 219 url_without_fragment = url.ReplaceComponents(f_replacements); | |
| 220 | |
| 221 // 2. Do URL unescaping until no more hex encoded characters exist. | |
| 222 std::string url_unescaped_str(Unescape(url_without_fragment.spec())); | |
| 223 url::Parsed parsed; | |
| 224 url::ParseStandardURL(url_unescaped_str.data(), url_unescaped_str.length(), | |
| 225 &parsed); | |
| 226 | |
| 227 // 3. In hostname, remove all leading and trailing dots. | |
| 228 const std::string host = | |
| 229 (parsed.host.len > 0) | |
| 230 ? url_unescaped_str.substr(parsed.host.begin, parsed.host.len) | |
| 231 : std::string(); | |
| 232 std::string host_without_end_dots; | |
| 233 base::TrimString(host, ".", &host_without_end_dots); | |
| 234 | |
| 235 // 4. In hostname, replace consecutive dots with a single dot. | |
| 236 std::string host_without_consecutive_dots(RemoveConsecutiveChars( | |
| 237 host_without_end_dots, '.')); | |
| 238 | |
| 239 // 5. In path, replace runs of consecutive slashes with a single slash. | |
| 240 std::string path = | |
| 241 (parsed.path.len > 0) | |
| 242 ? url_unescaped_str.substr(parsed.path.begin, parsed.path.len) | |
| 243 : std::string(); | |
| 244 std::string path_without_consecutive_slash(RemoveConsecutiveChars(path, '/')); | |
| 245 | |
| 246 url::Replacements<char> hp_replacements; | |
| 247 hp_replacements.SetHost( | |
| 248 host_without_consecutive_dots.data(), | |
| 249 url::Component(0, host_without_consecutive_dots.length())); | |
| 250 hp_replacements.SetPath( | |
| 251 path_without_consecutive_slash.data(), | |
| 252 url::Component(0, path_without_consecutive_slash.length())); | |
| 253 | |
| 254 std::string url_unescaped_with_can_hostpath; | |
| 255 url::StdStringCanonOutput output(&url_unescaped_with_can_hostpath); | |
| 256 url::Parsed temp_parsed; | |
| 257 url::ReplaceComponents(url_unescaped_str.data(), | |
| 258 url_unescaped_str.length(), | |
| 259 parsed, | |
| 260 hp_replacements, | |
| 261 NULL, | |
| 262 &output, | |
| 263 &temp_parsed); | |
| 264 output.Complete(); | |
| 265 | |
| 266 // 6. Step needed to revert escaping done in url::ReplaceComponents. | |
| 267 url_unescaped_with_can_hostpath = Unescape(url_unescaped_with_can_hostpath); | |
| 268 | |
| 269 // 7. After performing all above steps, percent-escape all chars in url which | |
| 270 // are <= ASCII 32, >= 127, #, %. Escapes must be uppercase hex characters. | |
| 271 std::string escaped_canon_url_str(Escape(url_unescaped_with_can_hostpath)); | |
| 272 url::Parsed final_parsed; | |
| 273 url::ParseStandardURL(escaped_canon_url_str.data(), | |
| 274 escaped_canon_url_str.length(), | |
| 275 &final_parsed); | |
| 276 | |
| 277 if (canonicalized_hostname && final_parsed.host.len > 0) { | |
| 278 *canonicalized_hostname = | |
| 279 escaped_canon_url_str.substr(final_parsed.host.begin, | |
| 280 final_parsed.host.len); | |
| 281 } | |
| 282 if (canonicalized_path && final_parsed.path.len > 0) { | |
| 283 *canonicalized_path = escaped_canon_url_str.substr(final_parsed.path.begin, | |
| 284 final_parsed.path.len); | |
| 285 } | |
| 286 if (canonicalized_query && final_parsed.query.len > 0) { | |
| 287 *canonicalized_query = escaped_canon_url_str.substr( | |
| 288 final_parsed.query.begin, final_parsed.query.len); | |
| 289 } | |
| 290 } | |
| 291 | |
| 292 void GenerateHostsToCheck(const GURL& url, std::vector<std::string>* hosts) { | |
| 293 hosts->clear(); | |
| 294 | |
| 295 std::string canon_host; | |
| 296 CanonicalizeUrl(url, &canon_host, NULL, NULL); | |
| 297 | |
| 298 const std::string host = canon_host; // const sidesteps GCC bugs below! | |
| 299 if (host.empty()) | |
| 300 return; | |
| 301 | |
| 302 // Per the Safe Browsing Protocol v2 spec, we try the host, and also up to 4 | |
| 303 // hostnames formed by starting with the last 5 components and successively | |
| 304 // removing the leading component. The last component isn't examined alone, | |
| 305 // since it's the TLD or a subcomponent thereof. | |
| 306 // | |
| 307 // Note that we don't need to be clever about stopping at the "real" eTLD -- | |
| 308 // the data on the server side has been filtered to ensure it will not | |
| 309 // blacklist a whole TLD, and it's not significantly slower on our side to | |
| 310 // just check too much. | |
| 311 // | |
| 312 // Also note that because we have a simple blacklist, not some sort of complex | |
| 313 // whitelist-in-blacklist or vice versa, it doesn't matter what order we check | |
| 314 // these in. | |
| 315 const size_t kMaxHostsToCheck = 4; | |
| 316 bool skipped_last_component = false; | |
| 317 for (std::string::const_reverse_iterator i(host.rbegin()); | |
| 318 i != host.rend() && hosts->size() < kMaxHostsToCheck; ++i) { | |
| 319 if (*i == '.') { | |
| 320 if (skipped_last_component) | |
| 321 hosts->push_back(std::string(i.base(), host.end())); | |
| 322 else | |
| 323 skipped_last_component = true; | |
| 324 } | |
| 325 } | |
| 326 hosts->push_back(host); | |
| 327 } | |
| 328 | |
| 329 void GeneratePathsToCheck(const GURL& url, std::vector<std::string>* paths) { | |
| 330 paths->clear(); | |
| 331 | |
| 332 std::string canon_path; | |
| 333 std::string canon_query; | |
| 334 CanonicalizeUrl(url, NULL, &canon_path, &canon_query); | |
| 335 | |
| 336 const std::string path = canon_path; // const sidesteps GCC bugs below! | |
| 337 const std::string query = canon_query; | |
| 338 if (path.empty()) | |
| 339 return; | |
| 340 | |
| 341 // Per the Safe Browsing Protocol v2 spec, we try the exact path with/without | |
| 342 // the query parameters, and also up to 4 paths formed by starting at the root | |
| 343 // and adding more path components. | |
| 344 // | |
| 345 // As with the hosts above, it doesn't matter what order we check these in. | |
| 346 const size_t kMaxPathsToCheck = 4; | |
| 347 for (std::string::const_iterator i(path.begin()); | |
| 348 i != path.end() && paths->size() < kMaxPathsToCheck; ++i) { | |
| 349 if (*i == '/') | |
| 350 paths->push_back(std::string(path.begin(), i + 1)); | |
| 351 } | |
| 352 | |
| 353 if (!paths->empty() && paths->back() != path) | |
| 354 paths->push_back(path); | |
| 355 | |
| 356 if (!query.empty()) | |
| 357 paths->push_back(path + "?" + query); | |
| 358 } | |
| 359 | |
| 360 void GeneratePatternsToCheck(const GURL& url, std::vector<std::string>* urls) { | |
| 361 std::vector<std::string> hosts, paths; | |
| 362 GenerateHostsToCheck(url, &hosts); | |
| 363 GeneratePathsToCheck(url, &paths); | |
| 364 for (size_t h = 0; h < hosts.size(); ++h) { | |
| 365 for (size_t p = 0; p < paths.size(); ++p) { | |
| 366 urls->push_back(hosts[h] + paths[p]); | |
| 367 } | |
| 368 } | |
| 369 } | |
| 370 | |
| 371 } // namespace safe_browsing | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1410343012:
Revert of Move more declarations from c/b/sb/sb_util.h to components. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@01_components