OLD | NEW |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "chrome/browser/safe_browsing/safe_browsing_util.h" | 5 #include "chrome/browser/safe_browsing/safe_browsing_util.h" |
6 | 6 |
7 #include "base/logging.h" | |
8 #include "base/strings/string_util.h" | 7 #include "base/strings/string_util.h" |
9 #include "base/strings/stringprintf.h" | 8 #include "base/strings/stringprintf.h" |
10 #include "chrome/browser/safe_browsing/chunk.pb.h" | 9 #include "chrome/browser/safe_browsing/chunk.pb.h" |
11 #include "components/google/core/browser/google_util.h" | 10 #include "components/google/core/browser/google_util.h" |
12 #include "crypto/sha2.h" | |
13 #include "net/base/escape.h" | |
14 #include "url/gurl.h" | |
15 #include "url/url_util.h" | |
16 | |
17 // SBCachedFullHashResult ------------------------------------------------------ | |
18 | |
19 SBCachedFullHashResult::SBCachedFullHashResult() {} | |
20 | |
21 SBCachedFullHashResult::SBCachedFullHashResult( | |
22 const base::Time& in_expire_after) | |
23 : expire_after(in_expire_after) {} | |
24 | |
25 SBCachedFullHashResult::~SBCachedFullHashResult() {} | |
26 | 11 |
27 // SBChunkData ----------------------------------------------------------------- | 12 // SBChunkData ----------------------------------------------------------------- |
28 | 13 |
29 // TODO(shess): Right now this contains a scoped_ptr<ChunkData> so that the | 14 // TODO(shess): Right now this contains a scoped_ptr<ChunkData> so that the |
30 // proto buffer isn't copied all over the place, then these are contained in a | 15 // proto buffer isn't copied all over the place, then these are contained in a |
31 // ScopedVector for purposes of passing things around between tasks. This seems | 16 // ScopedVector for purposes of passing things around between tasks. This seems |
32 // convoluted. Maybe it would make sense to have an overall container class | 17 // convoluted. Maybe it would make sense to have an overall container class |
33 // returning references to a nested per-chunk class? | 18 // returning references to a nested per-chunk class? |
34 | 19 |
35 SBChunkData::SBChunkData() { | 20 SBChunkData::SBChunkData() { |
(...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
136 | 121 |
137 SBListChunkRanges::SBListChunkRanges(const std::string& n) | 122 SBListChunkRanges::SBListChunkRanges(const std::string& n) |
138 : name(n) { | 123 : name(n) { |
139 } | 124 } |
140 | 125 |
141 // SBChunkDelete --------------------------------------------------------------- | 126 // SBChunkDelete --------------------------------------------------------------- |
142 | 127 |
143 SBChunkDelete::SBChunkDelete() : is_sub_del(false) {} | 128 SBChunkDelete::SBChunkDelete() : is_sub_del(false) {} |
144 | 129 |
145 SBChunkDelete::~SBChunkDelete() {} | 130 SBChunkDelete::~SBChunkDelete() {} |
146 | |
147 // Utility functions ----------------------------------------------------------- | |
148 | |
149 namespace { | |
150 bool IsKnownList(const std::string& name) { | |
151 for (size_t i = 0; i < arraysize(safe_browsing_util::kAllLists); ++i) { | |
152 if (!strcmp(safe_browsing_util::kAllLists[i], name.c_str())) { | |
153 return true; | |
154 } | |
155 } | |
156 return false; | |
157 } | |
158 } // namespace | |
159 | |
160 namespace safe_browsing_util { | |
161 | |
162 // Listnames that browser can process. | |
163 const char kMalwareList[] = "goog-malware-shavar"; | |
164 const char kPhishingList[] = "goog-phish-shavar"; | |
165 const char kBinUrlList[] = "goog-badbinurl-shavar"; | |
166 const char kCsdWhiteList[] = "goog-csdwhite-sha256"; | |
167 const char kDownloadWhiteList[] = "goog-downloadwhite-digest256"; | |
168 const char kExtensionBlacklist[] = "goog-badcrxids-digestvar"; | |
169 const char kIPBlacklist[] = "goog-badip-digest256"; | |
170 const char kUnwantedUrlList[] = "goog-unwanted-shavar"; | |
171 const char kInclusionWhitelist[] = "goog-csdinclusionwhite-sha256"; | |
172 | |
173 const char* kAllLists[9] = { | |
174 kMalwareList, | |
175 kPhishingList, | |
176 kBinUrlList, | |
177 kCsdWhiteList, | |
178 kDownloadWhiteList, | |
179 kExtensionBlacklist, | |
180 kIPBlacklist, | |
181 kUnwantedUrlList, | |
182 kInclusionWhitelist, | |
183 }; | |
184 | |
185 ListType GetListId(const base::StringPiece& name) { | |
186 ListType id; | |
187 if (name == safe_browsing_util::kMalwareList) { | |
188 id = MALWARE; | |
189 } else if (name == safe_browsing_util::kPhishingList) { | |
190 id = PHISH; | |
191 } else if (name == safe_browsing_util::kBinUrlList) { | |
192 id = BINURL; | |
193 } else if (name == safe_browsing_util::kCsdWhiteList) { | |
194 id = CSDWHITELIST; | |
195 } else if (name == safe_browsing_util::kDownloadWhiteList) { | |
196 id = DOWNLOADWHITELIST; | |
197 } else if (name == safe_browsing_util::kExtensionBlacklist) { | |
198 id = EXTENSIONBLACKLIST; | |
199 } else if (name == safe_browsing_util::kIPBlacklist) { | |
200 id = IPBLACKLIST; | |
201 } else if (name == safe_browsing_util::kUnwantedUrlList) { | |
202 id = UNWANTEDURL; | |
203 } else if (name == safe_browsing_util::kInclusionWhitelist) { | |
204 id = INCLUSIONWHITELIST; | |
205 } else { | |
206 id = INVALID; | |
207 } | |
208 return id; | |
209 } | |
210 | |
211 bool GetListName(ListType list_id, std::string* list) { | |
212 switch (list_id) { | |
213 case MALWARE: | |
214 *list = safe_browsing_util::kMalwareList; | |
215 break; | |
216 case PHISH: | |
217 *list = safe_browsing_util::kPhishingList; | |
218 break; | |
219 case BINURL: | |
220 *list = safe_browsing_util::kBinUrlList; | |
221 break; | |
222 case CSDWHITELIST: | |
223 *list = safe_browsing_util::kCsdWhiteList; | |
224 break; | |
225 case DOWNLOADWHITELIST: | |
226 *list = safe_browsing_util::kDownloadWhiteList; | |
227 break; | |
228 case EXTENSIONBLACKLIST: | |
229 *list = safe_browsing_util::kExtensionBlacklist; | |
230 break; | |
231 case IPBLACKLIST: | |
232 *list = safe_browsing_util::kIPBlacklist; | |
233 break; | |
234 case UNWANTEDURL: | |
235 *list = safe_browsing_util::kUnwantedUrlList; | |
236 break; | |
237 case INCLUSIONWHITELIST: | |
238 *list = safe_browsing_util::kInclusionWhitelist; | |
239 break; | |
240 default: | |
241 return false; | |
242 } | |
243 DCHECK(IsKnownList(*list)); | |
244 return true; | |
245 } | |
246 | |
247 std::string Unescape(const std::string& url) { | |
248 std::string unescaped_str(url); | |
249 std::string old_unescaped_str; | |
250 const int kMaxLoopIterations = 1024; | |
251 int loop_var = 0; | |
252 do { | |
253 old_unescaped_str = unescaped_str; | |
254 unescaped_str = net::UnescapeURLComponent( | |
255 old_unescaped_str, net::UnescapeRule::SPOOFING_AND_CONTROL_CHARS | | |
256 net::UnescapeRule::SPACES | | |
257 net::UnescapeRule::URL_SPECIAL_CHARS); | |
258 } while (unescaped_str != old_unescaped_str && ++loop_var <= | |
259 kMaxLoopIterations); | |
260 | |
261 return unescaped_str; | |
262 } | |
263 | |
264 std::string Escape(const std::string& url) { | |
265 std::string escaped_str; | |
266 const char* kHexString = "0123456789ABCDEF"; | |
267 for (size_t i = 0; i < url.length(); i++) { | |
268 unsigned char c = static_cast<unsigned char>(url[i]); | |
269 if (c <= ' ' || c > '~' || c == '#' || c == '%') { | |
270 escaped_str.push_back('%'); | |
271 escaped_str.push_back(kHexString[c >> 4]); | |
272 escaped_str.push_back(kHexString[c & 0xf]); | |
273 } else { | |
274 escaped_str.push_back(c); | |
275 } | |
276 } | |
277 | |
278 return escaped_str; | |
279 } | |
280 | |
281 std::string RemoveConsecutiveChars(const std::string& str, const char c) { | |
282 std::string output(str); | |
283 std::string string_to_find; | |
284 std::string::size_type loc = 0; | |
285 string_to_find.append(2, c); | |
286 while ((loc = output.find(string_to_find, loc)) != std::string::npos) { | |
287 output.erase(loc, 1); | |
288 } | |
289 | |
290 return output; | |
291 } | |
292 | |
293 // Canonicalizes url as per Google Safe Browsing Specification. | |
294 // See section 6.1 in | |
295 // http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec. | |
296 void CanonicalizeUrl(const GURL& url, | |
297 std::string* canonicalized_hostname, | |
298 std::string* canonicalized_path, | |
299 std::string* canonicalized_query) { | |
300 DCHECK(url.is_valid()); | |
301 | |
302 // We only canonicalize "normal" URLs. | |
303 if (!url.IsStandard()) | |
304 return; | |
305 | |
306 // Following canonicalization steps are excluded since url parsing takes care | |
307 // of those :- | |
308 // 1. Remove any tab (0x09), CR (0x0d), and LF (0x0a) chars from url. | |
309 // (Exclude escaped version of these chars). | |
310 // 2. Normalize hostname to 4 dot-seperated decimal values. | |
311 // 3. Lowercase hostname. | |
312 // 4. Resolve path sequences "/../" and "/./". | |
313 | |
314 // That leaves us with the following :- | |
315 // 1. Remove fragment in URL. | |
316 GURL url_without_fragment; | |
317 GURL::Replacements f_replacements; | |
318 f_replacements.ClearRef(); | |
319 f_replacements.ClearUsername(); | |
320 f_replacements.ClearPassword(); | |
321 url_without_fragment = url.ReplaceComponents(f_replacements); | |
322 | |
323 // 2. Do URL unescaping until no more hex encoded characters exist. | |
324 std::string url_unescaped_str(Unescape(url_without_fragment.spec())); | |
325 url::Parsed parsed; | |
326 url::ParseStandardURL(url_unescaped_str.data(), url_unescaped_str.length(), | |
327 &parsed); | |
328 | |
329 // 3. In hostname, remove all leading and trailing dots. | |
330 const std::string host = | |
331 (parsed.host.len > 0) | |
332 ? url_unescaped_str.substr(parsed.host.begin, parsed.host.len) | |
333 : std::string(); | |
334 std::string host_without_end_dots; | |
335 base::TrimString(host, ".", &host_without_end_dots); | |
336 | |
337 // 4. In hostname, replace consecutive dots with a single dot. | |
338 std::string host_without_consecutive_dots(RemoveConsecutiveChars( | |
339 host_without_end_dots, '.')); | |
340 | |
341 // 5. In path, replace runs of consecutive slashes with a single slash. | |
342 std::string path = | |
343 (parsed.path.len > 0) | |
344 ? url_unescaped_str.substr(parsed.path.begin, parsed.path.len) | |
345 : std::string(); | |
346 std::string path_without_consecutive_slash(RemoveConsecutiveChars(path, '/')); | |
347 | |
348 url::Replacements<char> hp_replacements; | |
349 hp_replacements.SetHost( | |
350 host_without_consecutive_dots.data(), | |
351 url::Component(0, host_without_consecutive_dots.length())); | |
352 hp_replacements.SetPath( | |
353 path_without_consecutive_slash.data(), | |
354 url::Component(0, path_without_consecutive_slash.length())); | |
355 | |
356 std::string url_unescaped_with_can_hostpath; | |
357 url::StdStringCanonOutput output(&url_unescaped_with_can_hostpath); | |
358 url::Parsed temp_parsed; | |
359 url::ReplaceComponents(url_unescaped_str.data(), | |
360 url_unescaped_str.length(), | |
361 parsed, | |
362 hp_replacements, | |
363 NULL, | |
364 &output, | |
365 &temp_parsed); | |
366 output.Complete(); | |
367 | |
368 // 6. Step needed to revert escaping done in url::ReplaceComponents. | |
369 url_unescaped_with_can_hostpath = Unescape(url_unescaped_with_can_hostpath); | |
370 | |
371 // 7. After performing all above steps, percent-escape all chars in url which | |
372 // are <= ASCII 32, >= 127, #, %. Escapes must be uppercase hex characters. | |
373 std::string escaped_canon_url_str(Escape(url_unescaped_with_can_hostpath)); | |
374 url::Parsed final_parsed; | |
375 url::ParseStandardURL(escaped_canon_url_str.data(), | |
376 escaped_canon_url_str.length(), | |
377 &final_parsed); | |
378 | |
379 if (canonicalized_hostname && final_parsed.host.len > 0) { | |
380 *canonicalized_hostname = | |
381 escaped_canon_url_str.substr(final_parsed.host.begin, | |
382 final_parsed.host.len); | |
383 } | |
384 if (canonicalized_path && final_parsed.path.len > 0) { | |
385 *canonicalized_path = escaped_canon_url_str.substr(final_parsed.path.begin, | |
386 final_parsed.path.len); | |
387 } | |
388 if (canonicalized_query && final_parsed.query.len > 0) { | |
389 *canonicalized_query = escaped_canon_url_str.substr( | |
390 final_parsed.query.begin, final_parsed.query.len); | |
391 } | |
392 } | |
393 | |
394 void GenerateHostsToCheck(const GURL& url, std::vector<std::string>* hosts) { | |
395 hosts->clear(); | |
396 | |
397 std::string canon_host; | |
398 CanonicalizeUrl(url, &canon_host, NULL, NULL); | |
399 | |
400 const std::string host = canon_host; // const sidesteps GCC bugs below! | |
401 if (host.empty()) | |
402 return; | |
403 | |
404 // Per the Safe Browsing Protocol v2 spec, we try the host, and also up to 4 | |
405 // hostnames formed by starting with the last 5 components and successively | |
406 // removing the leading component. The last component isn't examined alone, | |
407 // since it's the TLD or a subcomponent thereof. | |
408 // | |
409 // Note that we don't need to be clever about stopping at the "real" eTLD -- | |
410 // the data on the server side has been filtered to ensure it will not | |
411 // blacklist a whole TLD, and it's not significantly slower on our side to | |
412 // just check too much. | |
413 // | |
414 // Also note that because we have a simple blacklist, not some sort of complex | |
415 // whitelist-in-blacklist or vice versa, it doesn't matter what order we check | |
416 // these in. | |
417 const size_t kMaxHostsToCheck = 4; | |
418 bool skipped_last_component = false; | |
419 for (std::string::const_reverse_iterator i(host.rbegin()); | |
420 i != host.rend() && hosts->size() < kMaxHostsToCheck; ++i) { | |
421 if (*i == '.') { | |
422 if (skipped_last_component) | |
423 hosts->push_back(std::string(i.base(), host.end())); | |
424 else | |
425 skipped_last_component = true; | |
426 } | |
427 } | |
428 hosts->push_back(host); | |
429 } | |
430 | |
431 void GeneratePathsToCheck(const GURL& url, std::vector<std::string>* paths) { | |
432 paths->clear(); | |
433 | |
434 std::string canon_path; | |
435 std::string canon_query; | |
436 CanonicalizeUrl(url, NULL, &canon_path, &canon_query); | |
437 | |
438 const std::string path = canon_path; // const sidesteps GCC bugs below! | |
439 const std::string query = canon_query; | |
440 if (path.empty()) | |
441 return; | |
442 | |
443 // Per the Safe Browsing Protocol v2 spec, we try the exact path with/without | |
444 // the query parameters, and also up to 4 paths formed by starting at the root | |
445 // and adding more path components. | |
446 // | |
447 // As with the hosts above, it doesn't matter what order we check these in. | |
448 const size_t kMaxPathsToCheck = 4; | |
449 for (std::string::const_iterator i(path.begin()); | |
450 i != path.end() && paths->size() < kMaxPathsToCheck; ++i) { | |
451 if (*i == '/') | |
452 paths->push_back(std::string(path.begin(), i + 1)); | |
453 } | |
454 | |
455 if (!paths->empty() && paths->back() != path) | |
456 paths->push_back(path); | |
457 | |
458 if (!query.empty()) | |
459 paths->push_back(path + "?" + query); | |
460 } | |
461 | |
462 void GeneratePatternsToCheck(const GURL& url, std::vector<std::string>* urls) { | |
463 std::vector<std::string> hosts, paths; | |
464 GenerateHostsToCheck(url, &hosts); | |
465 GeneratePathsToCheck(url, &paths); | |
466 for (size_t h = 0; h < hosts.size(); ++h) { | |
467 for (size_t p = 0; p < paths.size(); ++p) { | |
468 urls->push_back(hosts[h] + paths[p]); | |
469 } | |
470 } | |
471 } | |
472 | |
473 SBFullHash StringToSBFullHash(const std::string& hash_in) { | |
474 DCHECK_EQ(crypto::kSHA256Length, hash_in.size()); | |
475 SBFullHash hash_out; | |
476 memcpy(hash_out.full_hash, hash_in.data(), crypto::kSHA256Length); | |
477 return hash_out; | |
478 } | |
479 | |
480 std::string SBFullHashToString(const SBFullHash& hash) { | |
481 DCHECK_EQ(crypto::kSHA256Length, sizeof(hash.full_hash)); | |
482 return std::string(hash.full_hash, sizeof(hash.full_hash)); | |
483 } | |
484 | |
485 } // namespace safe_browsing_util | |
OLD | NEW |