Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(329)

Side by Side Diff: components/safe_browsing_db/safe_browsing_db_util.cc

Issue 1420123003: Move more declarations from c/b/sb/sb_util.h to components. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@01_components
Patch Set: Making 1415923004 the base issue, instead of 1399843003 which was reverted. Created 5 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2015 The Chromium Authors. All rights reserved. 1 // Copyright (c) 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "components/safe_browsing_db/safe_browsing_db_util.h" 5 #include "components/safe_browsing_db/safe_browsing_db_util.h"
6 6
7 #include "base/strings/string_util.h"
7 #include "crypto/sha2.h" 8 #include "crypto/sha2.h"
9 #include "net/base/escape.h"
10 #include "url/gurl.h"
11 #include "url/url_util.h"
12
13 // Utility functions -----------------------------------------------------------
14
15 namespace {
16 bool IsKnownList(const std::string& name) {
17 for (size_t i = 0; i < arraysize(safe_browsing::kAllLists); ++i) {
18 if (!strcmp(safe_browsing::kAllLists[i], name.c_str())) {
19 return true;
20 }
21 }
22 return false;
23 }
24 } // namespace
25
26
27 // SBCachedFullHashResult ------------------------------------------------------
28
29 SBCachedFullHashResult::SBCachedFullHashResult() {}
30
31 SBCachedFullHashResult::SBCachedFullHashResult(
32 const base::Time& in_expire_after)
33 : expire_after(in_expire_after) {}
34
35 SBCachedFullHashResult::~SBCachedFullHashResult() {}
36
37
38 namespace safe_browsing {
39
40 // Listnames that browser can process.
41 // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
42 #if defined(OS_ANDROID)
43 // NOTE(shess): This difference is also reflected in the store name in
44 // safe_browsing_database.cc.
45 const char kMalwareList[] = "goog-mobilemalware-shavar";
Nathan Parker 2015/10/26 19:43:19 The OS_ANDROID section was removed (about a week a
46 const char kPhishingList[] = "goog-mobilephish-shavar";
47 #else
48 const char kMalwareList[] = "goog-malware-shavar";
49 const char kPhishingList[] = "goog-phish-shavar";
50 #endif
51 const char kBinUrlList[] = "goog-badbinurl-shavar";
52 const char kCsdWhiteList[] = "goog-csdwhite-sha256";
53 const char kDownloadWhiteList[] = "goog-downloadwhite-digest256";
54 const char kExtensionBlacklist[] = "goog-badcrxids-digestvar";
55 const char kIPBlacklist[] = "goog-badip-digest256";
56 const char kUnwantedUrlList[] = "goog-unwanted-shavar";
57 const char kInclusionWhitelist[] = "goog-csdinclusionwhite-sha256";
58
59 const char* kAllLists[9] = {
60 kMalwareList,
61 kPhishingList,
62 kBinUrlList,
63 kCsdWhiteList,
64 kDownloadWhiteList,
65 kExtensionBlacklist,
66 kIPBlacklist,
67 kUnwantedUrlList,
68 kInclusionWhitelist,
69 };
70
71 ListType GetListId(const base::StringPiece& name) {
72 ListType id;
73 if (name == kMalwareList) {
74 id = MALWARE;
75 } else if (name == kPhishingList) {
76 id = PHISH;
77 } else if (name == kBinUrlList) {
78 id = BINURL;
79 } else if (name == kCsdWhiteList) {
80 id = CSDWHITELIST;
81 } else if (name == kDownloadWhiteList) {
82 id = DOWNLOADWHITELIST;
83 } else if (name == kExtensionBlacklist) {
84 id = EXTENSIONBLACKLIST;
85 } else if (name == kIPBlacklist) {
86 id = IPBLACKLIST;
87 } else if (name == kUnwantedUrlList) {
88 id = UNWANTEDURL;
89 } else if (name == kInclusionWhitelist) {
90 id = INCLUSIONWHITELIST;
91 } else {
92 id = INVALID;
93 }
94 return id;
95 }
96
97 bool GetListName(ListType list_id, std::string* list) {
98 switch (list_id) {
99 case MALWARE:
100 *list = kMalwareList;
101 break;
102 case PHISH:
103 *list = kPhishingList;
104 break;
105 case BINURL:
106 *list = kBinUrlList;
107 break;
108 case CSDWHITELIST:
109 *list = kCsdWhiteList;
110 break;
111 case DOWNLOADWHITELIST:
112 *list = kDownloadWhiteList;
113 break;
114 case EXTENSIONBLACKLIST:
115 *list = kExtensionBlacklist;
116 break;
117 case IPBLACKLIST:
118 *list = kIPBlacklist;
119 break;
120 case UNWANTEDURL:
121 *list = kUnwantedUrlList;
122 break;
123 case INCLUSIONWHITELIST:
124 *list = kInclusionWhitelist;
125 break;
126 default:
127 return false;
128 }
129 DCHECK(IsKnownList(*list));
130 return true;
131 }
132
8 133
9 SBFullHash SBFullHashForString(const base::StringPiece& str) { 134 SBFullHash SBFullHashForString(const base::StringPiece& str) {
10 SBFullHash h; 135 SBFullHash h;
11 crypto::SHA256HashString(str, &h.full_hash, sizeof(h.full_hash)); 136 crypto::SHA256HashString(str, &h.full_hash, sizeof(h.full_hash));
12 return h; 137 return h;
13 } 138 }
139
140 SBFullHash StringToSBFullHash(const std::string& hash_in) {
141 DCHECK_EQ(crypto::kSHA256Length, hash_in.size());
142 SBFullHash hash_out;
143 memcpy(hash_out.full_hash, hash_in.data(), crypto::kSHA256Length);
144 return hash_out;
145 }
146
147 std::string SBFullHashToString(const SBFullHash& hash) {
148 DCHECK_EQ(crypto::kSHA256Length, sizeof(hash.full_hash));
149 return std::string(hash.full_hash, sizeof(hash.full_hash));
150 }
151
152
153 std::string Unescape(const std::string& url) {
154 std::string unescaped_str(url);
155 std::string old_unescaped_str;
156 const int kMaxLoopIterations = 1024;
157 int loop_var = 0;
158 do {
159 old_unescaped_str = unescaped_str;
160 unescaped_str = net::UnescapeURLComponent(
161 old_unescaped_str, net::UnescapeRule::SPOOFING_AND_CONTROL_CHARS |
162 net::UnescapeRule::SPACES |
163 net::UnescapeRule::URL_SPECIAL_CHARS);
164 } while (unescaped_str != old_unescaped_str && ++loop_var <=
165 kMaxLoopIterations);
166
167 return unescaped_str;
168 }
169
170 std::string Escape(const std::string& url) {
171 std::string escaped_str;
172 const char* kHexString = "0123456789ABCDEF";
173 for (size_t i = 0; i < url.length(); i++) {
174 unsigned char c = static_cast<unsigned char>(url[i]);
175 if (c <= ' ' || c > '~' || c == '#' || c == '%') {
176 escaped_str.push_back('%');
177 escaped_str.push_back(kHexString[c >> 4]);
178 escaped_str.push_back(kHexString[c & 0xf]);
179 } else {
180 escaped_str.push_back(c);
181 }
182 }
183
184 return escaped_str;
185 }
186
187 std::string RemoveConsecutiveChars(const std::string& str, const char c) {
188 std::string output(str);
189 std::string string_to_find;
190 std::string::size_type loc = 0;
191 string_to_find.append(2, c);
192 while ((loc = output.find(string_to_find, loc)) != std::string::npos) {
193 output.erase(loc, 1);
194 }
195
196 return output;
197 }
198
199 // Canonicalizes url as per Google Safe Browsing Specification.
200 // See section 6.1 in
201 // http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec.
202 void CanonicalizeUrl(const GURL& url,
203 std::string* canonicalized_hostname,
204 std::string* canonicalized_path,
205 std::string* canonicalized_query) {
206 DCHECK(url.is_valid());
207
208 // We only canonicalize "normal" URLs.
209 if (!url.IsStandard())
210 return;
211
212 // Following canonicalization steps are excluded since url parsing takes care
213 // of those :-
214 // 1. Remove any tab (0x09), CR (0x0d), and LF (0x0a) chars from url.
215 // (Exclude escaped version of these chars).
216 // 2. Normalize hostname to 4 dot-seperated decimal values.
217 // 3. Lowercase hostname.
218 // 4. Resolve path sequences "/../" and "/./".
219
220 // That leaves us with the following :-
221 // 1. Remove fragment in URL.
222 GURL url_without_fragment;
223 GURL::Replacements f_replacements;
224 f_replacements.ClearRef();
225 f_replacements.ClearUsername();
226 f_replacements.ClearPassword();
227 url_without_fragment = url.ReplaceComponents(f_replacements);
228
229 // 2. Do URL unescaping until no more hex encoded characters exist.
230 std::string url_unescaped_str(Unescape(url_without_fragment.spec()));
231 url::Parsed parsed;
232 url::ParseStandardURL(url_unescaped_str.data(), url_unescaped_str.length(),
233 &parsed);
234
235 // 3. In hostname, remove all leading and trailing dots.
236 const std::string host =
237 (parsed.host.len > 0)
238 ? url_unescaped_str.substr(parsed.host.begin, parsed.host.len)
239 : std::string();
240 std::string host_without_end_dots;
241 base::TrimString(host, ".", &host_without_end_dots);
242
243 // 4. In hostname, replace consecutive dots with a single dot.
244 std::string host_without_consecutive_dots(RemoveConsecutiveChars(
245 host_without_end_dots, '.'));
246
247 // 5. In path, replace runs of consecutive slashes with a single slash.
248 std::string path =
249 (parsed.path.len > 0)
250 ? url_unescaped_str.substr(parsed.path.begin, parsed.path.len)
251 : std::string();
252 std::string path_without_consecutive_slash(RemoveConsecutiveChars(path, '/'));
253
254 url::Replacements<char> hp_replacements;
255 hp_replacements.SetHost(
256 host_without_consecutive_dots.data(),
257 url::Component(0, host_without_consecutive_dots.length()));
258 hp_replacements.SetPath(
259 path_without_consecutive_slash.data(),
260 url::Component(0, path_without_consecutive_slash.length()));
261
262 std::string url_unescaped_with_can_hostpath;
263 url::StdStringCanonOutput output(&url_unescaped_with_can_hostpath);
264 url::Parsed temp_parsed;
265 url::ReplaceComponents(url_unescaped_str.data(),
266 url_unescaped_str.length(),
267 parsed,
268 hp_replacements,
269 NULL,
270 &output,
271 &temp_parsed);
272 output.Complete();
273
274 // 6. Step needed to revert escaping done in url::ReplaceComponents.
275 url_unescaped_with_can_hostpath = Unescape(url_unescaped_with_can_hostpath);
276
277 // 7. After performing all above steps, percent-escape all chars in url which
278 // are <= ASCII 32, >= 127, #, %. Escapes must be uppercase hex characters.
279 std::string escaped_canon_url_str(Escape(url_unescaped_with_can_hostpath));
280 url::Parsed final_parsed;
281 url::ParseStandardURL(escaped_canon_url_str.data(),
282 escaped_canon_url_str.length(),
283 &final_parsed);
284
285 if (canonicalized_hostname && final_parsed.host.len > 0) {
286 *canonicalized_hostname =
287 escaped_canon_url_str.substr(final_parsed.host.begin,
288 final_parsed.host.len);
289 }
290 if (canonicalized_path && final_parsed.path.len > 0) {
291 *canonicalized_path = escaped_canon_url_str.substr(final_parsed.path.begin,
292 final_parsed.path.len);
293 }
294 if (canonicalized_query && final_parsed.query.len > 0) {
295 *canonicalized_query = escaped_canon_url_str.substr(
296 final_parsed.query.begin, final_parsed.query.len);
297 }
298 }
299
300 void GenerateHostsToCheck(const GURL& url, std::vector<std::string>* hosts) {
301 hosts->clear();
302
303 std::string canon_host;
304 CanonicalizeUrl(url, &canon_host, NULL, NULL);
305
306 const std::string host = canon_host; // const sidesteps GCC bugs below!
307 if (host.empty())
308 return;
309
310 // Per the Safe Browsing Protocol v2 spec, we try the host, and also up to 4
311 // hostnames formed by starting with the last 5 components and successively
312 // removing the leading component. The last component isn't examined alone,
313 // since it's the TLD or a subcomponent thereof.
314 //
315 // Note that we don't need to be clever about stopping at the "real" eTLD --
316 // the data on the server side has been filtered to ensure it will not
317 // blacklist a whole TLD, and it's not significantly slower on our side to
318 // just check too much.
319 //
320 // Also note that because we have a simple blacklist, not some sort of complex
321 // whitelist-in-blacklist or vice versa, it doesn't matter what order we check
322 // these in.
323 const size_t kMaxHostsToCheck = 4;
324 bool skipped_last_component = false;
325 for (std::string::const_reverse_iterator i(host.rbegin());
326 i != host.rend() && hosts->size() < kMaxHostsToCheck; ++i) {
327 if (*i == '.') {
328 if (skipped_last_component)
329 hosts->push_back(std::string(i.base(), host.end()));
330 else
331 skipped_last_component = true;
332 }
333 }
334 hosts->push_back(host);
335 }
336
337 void GeneratePathsToCheck(const GURL& url, std::vector<std::string>* paths) {
338 paths->clear();
339
340 std::string canon_path;
341 std::string canon_query;
342 CanonicalizeUrl(url, NULL, &canon_path, &canon_query);
343
344 const std::string path = canon_path; // const sidesteps GCC bugs below!
345 const std::string query = canon_query;
346 if (path.empty())
347 return;
348
349 // Per the Safe Browsing Protocol v2 spec, we try the exact path with/without
350 // the query parameters, and also up to 4 paths formed by starting at the root
351 // and adding more path components.
352 //
353 // As with the hosts above, it doesn't matter what order we check these in.
354 const size_t kMaxPathsToCheck = 4;
355 for (std::string::const_iterator i(path.begin());
356 i != path.end() && paths->size() < kMaxPathsToCheck; ++i) {
357 if (*i == '/')
358 paths->push_back(std::string(path.begin(), i + 1));
359 }
360
361 if (!paths->empty() && paths->back() != path)
362 paths->push_back(path);
363
364 if (!query.empty())
365 paths->push_back(path + "?" + query);
366 }
367
368 void GeneratePatternsToCheck(const GURL& url, std::vector<std::string>* urls) {
369 std::vector<std::string> hosts, paths;
370 GenerateHostsToCheck(url, &hosts);
371 GeneratePathsToCheck(url, &paths);
372 for (size_t h = 0; h < hosts.size(); ++h) {
373 for (size_t p = 0; p < paths.size(); ++p) {
374 urls->push_back(hosts[h] + paths[p]);
375 }
376 }
377 }
378
379 } // namespace safe_browsing
OLDNEW
« no previous file with comments | « components/safe_browsing_db/safe_browsing_db_util.h ('k') | components/safe_browsing_db/safe_browsing_db_util_unittest.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698