OLD | NEW |
---|---|
1 // Copyright (c) 2015 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2015 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "components/safe_browsing_db/safe_browsing_db_util.h" | 5 #include "components/safe_browsing_db/safe_browsing_db_util.h" |
6 | 6 |
7 #include "base/strings/string_util.h" | |
7 #include "crypto/sha2.h" | 8 #include "crypto/sha2.h" |
9 #include "net/base/escape.h" | |
10 #include "url/gurl.h" | |
11 #include "url/url_util.h" | |
12 | |
13 // Utility functions ----------------------------------------------------------- | |
14 | |
15 namespace { | |
16 bool IsKnownList(const std::string& name) { | |
17 for (size_t i = 0; i < arraysize(safe_browsing::kAllLists); ++i) { | |
18 if (!strcmp(safe_browsing::kAllLists[i], name.c_str())) { | |
19 return true; | |
20 } | |
21 } | |
22 return false; | |
23 } | |
24 } // namespace | |
25 | |
26 | |
27 // SBCachedFullHashResult ------------------------------------------------------ | |
28 | |
29 SBCachedFullHashResult::SBCachedFullHashResult() {} | |
30 | |
31 SBCachedFullHashResult::SBCachedFullHashResult( | |
32 const base::Time& in_expire_after) | |
33 : expire_after(in_expire_after) {} | |
34 | |
35 SBCachedFullHashResult::~SBCachedFullHashResult() {} | |
36 | |
37 | |
38 namespace safe_browsing { | |
39 | |
40 // Listnames that browser can process. | |
41 // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379> | |
42 #if defined(OS_ANDROID) | |
43 // NOTE(shess): This difference is also reflected in the store name in | |
44 // safe_browsing_database.cc. | |
45 const char kMalwareList[] = "goog-mobilemalware-shavar"; | |
Nathan Parker
2015/10/26 19:43:19
The OS_ANDROID section was removed (about a week a
| |
46 const char kPhishingList[] = "goog-mobilephish-shavar"; | |
47 #else | |
48 const char kMalwareList[] = "goog-malware-shavar"; | |
49 const char kPhishingList[] = "goog-phish-shavar"; | |
50 #endif | |
51 const char kBinUrlList[] = "goog-badbinurl-shavar"; | |
52 const char kCsdWhiteList[] = "goog-csdwhite-sha256"; | |
53 const char kDownloadWhiteList[] = "goog-downloadwhite-digest256"; | |
54 const char kExtensionBlacklist[] = "goog-badcrxids-digestvar"; | |
55 const char kIPBlacklist[] = "goog-badip-digest256"; | |
56 const char kUnwantedUrlList[] = "goog-unwanted-shavar"; | |
57 const char kInclusionWhitelist[] = "goog-csdinclusionwhite-sha256"; | |
58 | |
59 const char* kAllLists[9] = { | |
60 kMalwareList, | |
61 kPhishingList, | |
62 kBinUrlList, | |
63 kCsdWhiteList, | |
64 kDownloadWhiteList, | |
65 kExtensionBlacklist, | |
66 kIPBlacklist, | |
67 kUnwantedUrlList, | |
68 kInclusionWhitelist, | |
69 }; | |
70 | |
71 ListType GetListId(const base::StringPiece& name) { | |
72 ListType id; | |
73 if (name == kMalwareList) { | |
74 id = MALWARE; | |
75 } else if (name == kPhishingList) { | |
76 id = PHISH; | |
77 } else if (name == kBinUrlList) { | |
78 id = BINURL; | |
79 } else if (name == kCsdWhiteList) { | |
80 id = CSDWHITELIST; | |
81 } else if (name == kDownloadWhiteList) { | |
82 id = DOWNLOADWHITELIST; | |
83 } else if (name == kExtensionBlacklist) { | |
84 id = EXTENSIONBLACKLIST; | |
85 } else if (name == kIPBlacklist) { | |
86 id = IPBLACKLIST; | |
87 } else if (name == kUnwantedUrlList) { | |
88 id = UNWANTEDURL; | |
89 } else if (name == kInclusionWhitelist) { | |
90 id = INCLUSIONWHITELIST; | |
91 } else { | |
92 id = INVALID; | |
93 } | |
94 return id; | |
95 } | |
96 | |
97 bool GetListName(ListType list_id, std::string* list) { | |
98 switch (list_id) { | |
99 case MALWARE: | |
100 *list = kMalwareList; | |
101 break; | |
102 case PHISH: | |
103 *list = kPhishingList; | |
104 break; | |
105 case BINURL: | |
106 *list = kBinUrlList; | |
107 break; | |
108 case CSDWHITELIST: | |
109 *list = kCsdWhiteList; | |
110 break; | |
111 case DOWNLOADWHITELIST: | |
112 *list = kDownloadWhiteList; | |
113 break; | |
114 case EXTENSIONBLACKLIST: | |
115 *list = kExtensionBlacklist; | |
116 break; | |
117 case IPBLACKLIST: | |
118 *list = kIPBlacklist; | |
119 break; | |
120 case UNWANTEDURL: | |
121 *list = kUnwantedUrlList; | |
122 break; | |
123 case INCLUSIONWHITELIST: | |
124 *list = kInclusionWhitelist; | |
125 break; | |
126 default: | |
127 return false; | |
128 } | |
129 DCHECK(IsKnownList(*list)); | |
130 return true; | |
131 } | |
132 | |
8 | 133 |
9 SBFullHash SBFullHashForString(const base::StringPiece& str) { | 134 SBFullHash SBFullHashForString(const base::StringPiece& str) { |
10 SBFullHash h; | 135 SBFullHash h; |
11 crypto::SHA256HashString(str, &h.full_hash, sizeof(h.full_hash)); | 136 crypto::SHA256HashString(str, &h.full_hash, sizeof(h.full_hash)); |
12 return h; | 137 return h; |
13 } | 138 } |
139 | |
140 SBFullHash StringToSBFullHash(const std::string& hash_in) { | |
141 DCHECK_EQ(crypto::kSHA256Length, hash_in.size()); | |
142 SBFullHash hash_out; | |
143 memcpy(hash_out.full_hash, hash_in.data(), crypto::kSHA256Length); | |
144 return hash_out; | |
145 } | |
146 | |
147 std::string SBFullHashToString(const SBFullHash& hash) { | |
148 DCHECK_EQ(crypto::kSHA256Length, sizeof(hash.full_hash)); | |
149 return std::string(hash.full_hash, sizeof(hash.full_hash)); | |
150 } | |
151 | |
152 | |
153 std::string Unescape(const std::string& url) { | |
154 std::string unescaped_str(url); | |
155 std::string old_unescaped_str; | |
156 const int kMaxLoopIterations = 1024; | |
157 int loop_var = 0; | |
158 do { | |
159 old_unescaped_str = unescaped_str; | |
160 unescaped_str = net::UnescapeURLComponent( | |
161 old_unescaped_str, net::UnescapeRule::SPOOFING_AND_CONTROL_CHARS | | |
162 net::UnescapeRule::SPACES | | |
163 net::UnescapeRule::URL_SPECIAL_CHARS); | |
164 } while (unescaped_str != old_unescaped_str && ++loop_var <= | |
165 kMaxLoopIterations); | |
166 | |
167 return unescaped_str; | |
168 } | |
169 | |
170 std::string Escape(const std::string& url) { | |
171 std::string escaped_str; | |
172 const char* kHexString = "0123456789ABCDEF"; | |
173 for (size_t i = 0; i < url.length(); i++) { | |
174 unsigned char c = static_cast<unsigned char>(url[i]); | |
175 if (c <= ' ' || c > '~' || c == '#' || c == '%') { | |
176 escaped_str.push_back('%'); | |
177 escaped_str.push_back(kHexString[c >> 4]); | |
178 escaped_str.push_back(kHexString[c & 0xf]); | |
179 } else { | |
180 escaped_str.push_back(c); | |
181 } | |
182 } | |
183 | |
184 return escaped_str; | |
185 } | |
186 | |
187 std::string RemoveConsecutiveChars(const std::string& str, const char c) { | |
188 std::string output(str); | |
189 std::string string_to_find; | |
190 std::string::size_type loc = 0; | |
191 string_to_find.append(2, c); | |
192 while ((loc = output.find(string_to_find, loc)) != std::string::npos) { | |
193 output.erase(loc, 1); | |
194 } | |
195 | |
196 return output; | |
197 } | |
198 | |
199 // Canonicalizes url as per Google Safe Browsing Specification. | |
200 // See section 6.1 in | |
201 // http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec. | |
202 void CanonicalizeUrl(const GURL& url, | |
203 std::string* canonicalized_hostname, | |
204 std::string* canonicalized_path, | |
205 std::string* canonicalized_query) { | |
206 DCHECK(url.is_valid()); | |
207 | |
208 // We only canonicalize "normal" URLs. | |
209 if (!url.IsStandard()) | |
210 return; | |
211 | |
212 // Following canonicalization steps are excluded since url parsing takes care | |
213 // of those :- | |
214 // 1. Remove any tab (0x09), CR (0x0d), and LF (0x0a) chars from url. | |
215 // (Exclude escaped version of these chars). | |
216 // 2. Normalize hostname to 4 dot-seperated decimal values. | |
217 // 3. Lowercase hostname. | |
218 // 4. Resolve path sequences "/../" and "/./". | |
219 | |
220 // That leaves us with the following :- | |
221 // 1. Remove fragment in URL. | |
222 GURL url_without_fragment; | |
223 GURL::Replacements f_replacements; | |
224 f_replacements.ClearRef(); | |
225 f_replacements.ClearUsername(); | |
226 f_replacements.ClearPassword(); | |
227 url_without_fragment = url.ReplaceComponents(f_replacements); | |
228 | |
229 // 2. Do URL unescaping until no more hex encoded characters exist. | |
230 std::string url_unescaped_str(Unescape(url_without_fragment.spec())); | |
231 url::Parsed parsed; | |
232 url::ParseStandardURL(url_unescaped_str.data(), url_unescaped_str.length(), | |
233 &parsed); | |
234 | |
235 // 3. In hostname, remove all leading and trailing dots. | |
236 const std::string host = | |
237 (parsed.host.len > 0) | |
238 ? url_unescaped_str.substr(parsed.host.begin, parsed.host.len) | |
239 : std::string(); | |
240 std::string host_without_end_dots; | |
241 base::TrimString(host, ".", &host_without_end_dots); | |
242 | |
243 // 4. In hostname, replace consecutive dots with a single dot. | |
244 std::string host_without_consecutive_dots(RemoveConsecutiveChars( | |
245 host_without_end_dots, '.')); | |
246 | |
247 // 5. In path, replace runs of consecutive slashes with a single slash. | |
248 std::string path = | |
249 (parsed.path.len > 0) | |
250 ? url_unescaped_str.substr(parsed.path.begin, parsed.path.len) | |
251 : std::string(); | |
252 std::string path_without_consecutive_slash(RemoveConsecutiveChars(path, '/')); | |
253 | |
254 url::Replacements<char> hp_replacements; | |
255 hp_replacements.SetHost( | |
256 host_without_consecutive_dots.data(), | |
257 url::Component(0, host_without_consecutive_dots.length())); | |
258 hp_replacements.SetPath( | |
259 path_without_consecutive_slash.data(), | |
260 url::Component(0, path_without_consecutive_slash.length())); | |
261 | |
262 std::string url_unescaped_with_can_hostpath; | |
263 url::StdStringCanonOutput output(&url_unescaped_with_can_hostpath); | |
264 url::Parsed temp_parsed; | |
265 url::ReplaceComponents(url_unescaped_str.data(), | |
266 url_unescaped_str.length(), | |
267 parsed, | |
268 hp_replacements, | |
269 NULL, | |
270 &output, | |
271 &temp_parsed); | |
272 output.Complete(); | |
273 | |
274 // 6. Step needed to revert escaping done in url::ReplaceComponents. | |
275 url_unescaped_with_can_hostpath = Unescape(url_unescaped_with_can_hostpath); | |
276 | |
277 // 7. After performing all above steps, percent-escape all chars in url which | |
278 // are <= ASCII 32, >= 127, #, %. Escapes must be uppercase hex characters. | |
279 std::string escaped_canon_url_str(Escape(url_unescaped_with_can_hostpath)); | |
280 url::Parsed final_parsed; | |
281 url::ParseStandardURL(escaped_canon_url_str.data(), | |
282 escaped_canon_url_str.length(), | |
283 &final_parsed); | |
284 | |
285 if (canonicalized_hostname && final_parsed.host.len > 0) { | |
286 *canonicalized_hostname = | |
287 escaped_canon_url_str.substr(final_parsed.host.begin, | |
288 final_parsed.host.len); | |
289 } | |
290 if (canonicalized_path && final_parsed.path.len > 0) { | |
291 *canonicalized_path = escaped_canon_url_str.substr(final_parsed.path.begin, | |
292 final_parsed.path.len); | |
293 } | |
294 if (canonicalized_query && final_parsed.query.len > 0) { | |
295 *canonicalized_query = escaped_canon_url_str.substr( | |
296 final_parsed.query.begin, final_parsed.query.len); | |
297 } | |
298 } | |
299 | |
300 void GenerateHostsToCheck(const GURL& url, std::vector<std::string>* hosts) { | |
301 hosts->clear(); | |
302 | |
303 std::string canon_host; | |
304 CanonicalizeUrl(url, &canon_host, NULL, NULL); | |
305 | |
306 const std::string host = canon_host; // const sidesteps GCC bugs below! | |
307 if (host.empty()) | |
308 return; | |
309 | |
310 // Per the Safe Browsing Protocol v2 spec, we try the host, and also up to 4 | |
311 // hostnames formed by starting with the last 5 components and successively | |
312 // removing the leading component. The last component isn't examined alone, | |
313 // since it's the TLD or a subcomponent thereof. | |
314 // | |
315 // Note that we don't need to be clever about stopping at the "real" eTLD -- | |
316 // the data on the server side has been filtered to ensure it will not | |
317 // blacklist a whole TLD, and it's not significantly slower on our side to | |
318 // just check too much. | |
319 // | |
320 // Also note that because we have a simple blacklist, not some sort of complex | |
321 // whitelist-in-blacklist or vice versa, it doesn't matter what order we check | |
322 // these in. | |
323 const size_t kMaxHostsToCheck = 4; | |
324 bool skipped_last_component = false; | |
325 for (std::string::const_reverse_iterator i(host.rbegin()); | |
326 i != host.rend() && hosts->size() < kMaxHostsToCheck; ++i) { | |
327 if (*i == '.') { | |
328 if (skipped_last_component) | |
329 hosts->push_back(std::string(i.base(), host.end())); | |
330 else | |
331 skipped_last_component = true; | |
332 } | |
333 } | |
334 hosts->push_back(host); | |
335 } | |
336 | |
337 void GeneratePathsToCheck(const GURL& url, std::vector<std::string>* paths) { | |
338 paths->clear(); | |
339 | |
340 std::string canon_path; | |
341 std::string canon_query; | |
342 CanonicalizeUrl(url, NULL, &canon_path, &canon_query); | |
343 | |
344 const std::string path = canon_path; // const sidesteps GCC bugs below! | |
345 const std::string query = canon_query; | |
346 if (path.empty()) | |
347 return; | |
348 | |
349 // Per the Safe Browsing Protocol v2 spec, we try the exact path with/without | |
350 // the query parameters, and also up to 4 paths formed by starting at the root | |
351 // and adding more path components. | |
352 // | |
353 // As with the hosts above, it doesn't matter what order we check these in. | |
354 const size_t kMaxPathsToCheck = 4; | |
355 for (std::string::const_iterator i(path.begin()); | |
356 i != path.end() && paths->size() < kMaxPathsToCheck; ++i) { | |
357 if (*i == '/') | |
358 paths->push_back(std::string(path.begin(), i + 1)); | |
359 } | |
360 | |
361 if (!paths->empty() && paths->back() != path) | |
362 paths->push_back(path); | |
363 | |
364 if (!query.empty()) | |
365 paths->push_back(path + "?" + query); | |
366 } | |
367 | |
368 void GeneratePatternsToCheck(const GURL& url, std::vector<std::string>* urls) { | |
369 std::vector<std::string> hosts, paths; | |
370 GenerateHostsToCheck(url, &hosts); | |
371 GeneratePathsToCheck(url, &paths); | |
372 for (size_t h = 0; h < hosts.size(); ++h) { | |
373 for (size_t p = 0; p < paths.size(); ++p) { | |
374 urls->push_back(hosts[h] + paths[p]); | |
375 } | |
376 } | |
377 } | |
378 | |
379 } // namespace safe_browsing | |
OLD | NEW |