OLD | NEW |
---|---|
1 // Copyright 2016 The Chromium Authors. All rights reserved. | 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager .h" | 5 #include "chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager .h" |
6 | 6 |
7 #include "base/memory/ptr_util.h" | 7 #include "base/memory/ptr_util.h" |
8 #include "base/metrics/histogram_macros.h" | |
9 #include "base/strings/stringprintf.h" | |
8 #include "base/time/time.h" | 10 #include "base/time/time.h" |
11 #include "base/timer/timer.h" | |
9 #include "chrome/browser/chrome_notification_types.h" | 12 #include "chrome/browser/chrome_notification_types.h" |
10 #include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h" | 13 #include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h" |
11 #include "chrome/browser/sessions/session_tab_helper.h" | 14 #include "chrome/browser/sessions/session_tab_helper.h" |
12 #include "chrome/browser/tab_contents/retargeting_details.h" | 15 #include "chrome/browser/tab_contents/retargeting_details.h" |
13 #include "content/public/browser/navigation_details.h" | 16 #include "content/public/browser/navigation_details.h" |
14 #include "content/public/browser/notification_service.h" | 17 #include "content/public/browser/notification_service.h" |
15 #include "content/public/browser/notification_types.h" | 18 #include "content/public/browser/notification_types.h" |
16 #include "content/public/browser/render_frame_host.h" | 19 #include "content/public/browser/render_frame_host.h" |
17 #include "content/public/browser/render_process_host.h" | 20 #include "content/public/browser/render_process_host.h" |
18 #include "content/public/browser/web_contents.h" | 21 #include "content/public/browser/web_contents.h" |
19 | 22 |
20 using content::WebContents; | 23 using content::WebContents; |
21 | 24 |
22 namespace safe_browsing { | 25 namespace safe_browsing { |
23 | 26 |
27 namespace { | |
28 | |
29 // Given when an event happened and its TTL, determine if it is already expired. | |
30 // Note, if for some reason this event's timestamp is in the future, this | |
31 // event's timestamp is invalid, hence we treat it as expired. | |
32 bool IsEventExpired(const base::Time& event_time, double ttl_in_second) { | |
33 double current_time_in_second = base::Time::Now().ToDoubleT(); | |
34 double event_time_in_second = event_time.ToDoubleT(); | |
35 if (current_time_in_second <= event_time_in_second) | |
36 return true; | |
37 return current_time_in_second - event_time_in_second > ttl_in_second; | |
38 } | |
39 | |
40 } // namespace | |
41 | |
24 // The expiration period of a user gesture. Any user gesture that happened 1.0 | 42 // The expiration period of a user gesture. Any user gesture that happened 1.0 |
25 // second ago will be considered as expired and not relevant to upcoming | 43 // second ago is considered as expired and not relevant to upcoming navigation |
26 // navigation events. | 44 // events. |
27 static const double kUserGestureTTLInSecond = 1.0; | 45 static const double kUserGestureTTLInSecond = 1.0; |
46 // The expiration period of navigation events and resolved IP addresses. Any | |
47 // navigation related records that happened 2 minutes ago are considered as | |
48 // expired. So we clean up these navigation footprints every 2 minutes. | |
49 static const double kNavigationFootprintTTLInSecond = 120.0; | |
50 // The number of user gestures we trace back for download attribution. | |
51 static const int kDownloadAttributionUserGestureLimit = 2; | |
28 | 52 |
29 // static | 53 // static |
30 bool SafeBrowsingNavigationObserverManager::IsUserGestureExpired( | 54 bool SafeBrowsingNavigationObserverManager::IsUserGestureExpired( |
31 const base::Time& timestamp) { | 55 const base::Time& timestamp) { |
32 double now = base::Time::Now().ToDoubleT(); | 56 return IsEventExpired(timestamp, kUserGestureTTLInSecond); |
33 double timestamp_in_double = timestamp.ToDoubleT(); | |
34 | |
35 if (now <= timestamp_in_double) | |
36 return true; | |
37 return (now - timestamp_in_double) > kUserGestureTTLInSecond; | |
38 } | 57 } |
39 | 58 |
40 // static | 59 // static |
41 GURL SafeBrowsingNavigationObserverManager::ClearEmptyRef(const GURL& url) { | 60 GURL SafeBrowsingNavigationObserverManager::ClearEmptyRef(const GURL& url) { |
42 if (url.has_ref() && url.ref().empty()) { | 61 if (url.has_ref() && url.ref().empty()) { |
43 url::Replacements<char> replacements; | 62 url::Replacements<char> replacements; |
44 replacements.ClearRef(); | 63 replacements.ClearRef(); |
45 return url.ReplaceComponents(replacements); | 64 return url.ReplaceComponents(replacements); |
46 } | 65 } |
47 return url; | 66 return url; |
48 } | 67 } |
49 | 68 |
50 SafeBrowsingNavigationObserverManager::SafeBrowsingNavigationObserverManager() { | 69 SafeBrowsingNavigationObserverManager::SafeBrowsingNavigationObserverManager() { |
51 registrar_.Add(this, chrome::NOTIFICATION_RETARGETING, | 70 registrar_.Add(this, chrome::NOTIFICATION_RETARGETING, |
52 content::NotificationService::AllSources()); | 71 content::NotificationService::AllSources()); |
72 | |
73 // TODO(jialiul): call ScheduleNextCleanUpAfterInterval() when this class is | |
74 // ready to be hooked into SafeBrowsingService. | |
53 } | 75 } |
54 | 76 |
55 void SafeBrowsingNavigationObserverManager::RecordNavigationEvent( | 77 void SafeBrowsingNavigationObserverManager::RecordNavigationEvent( |
56 const GURL& nav_event_key, | 78 const GURL& nav_event_key, |
57 NavigationEvent* nav_event) { | 79 NavigationEvent* nav_event) { |
58 auto insertion_result = navigation_map_.insert( | 80 auto insertion_result = navigation_map_.insert( |
59 std::make_pair(nav_event_key, std::vector<NavigationEvent>())); | 81 std::make_pair(nav_event_key, std::vector<NavigationEvent>())); |
60 | 82 |
61 insertion_result.first->second.push_back(std::move(*nav_event)); | 83 insertion_result.first->second.push_back(std::move(*nav_event)); |
62 } | 84 } |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
97 } | 119 } |
98 } | 120 } |
99 // If this is a new IP of this host, and we added to the end of the vector. | 121 // If this is a new IP of this host, and we added to the end of the vector. |
100 insert_result.first->second.push_back( | 122 insert_result.first->second.push_back( |
101 ResolvedIPAddress(base::Time::Now(), ip)); | 123 ResolvedIPAddress(base::Time::Now(), ip)); |
102 } | 124 } |
103 | 125 |
104 void SafeBrowsingNavigationObserverManager::OnWebContentDestroyed( | 126 void SafeBrowsingNavigationObserverManager::OnWebContentDestroyed( |
105 content::WebContents* web_contents) { | 127 content::WebContents* web_contents) { |
106 user_gesture_map_.erase(web_contents); | 128 user_gesture_map_.erase(web_contents); |
107 // TODO (jialiul): Will add other clean up tasks shortly. | 129 } |
130 | |
131 void SafeBrowsingNavigationObserverManager::CleanUpStaleNavigationFootprints() { | |
132 CleanUpNavigationEvents(); | |
133 CleanUpUserGestures(); | |
134 CleanUpIpAddresses(); | |
135 ScheduleNextCleanUpAfterInterval( | |
136 base::TimeDelta::FromSecondsD(kNavigationFootprintTTLInSecond)); | |
137 } | |
138 | |
139 SafeBrowsingNavigationObserverManager::AttributionResult | |
140 SafeBrowsingNavigationObserverManager::IdentifyReferrerChain( | |
141 const GURL& target_url, | |
142 int target_tab_id, | |
143 int user_gesture_count_limit, | |
144 std::vector<ReferrerChainEntry>* out_referrer_chain) { | |
145 if (!target_url.is_valid()) | |
146 return INVALID_URL; | |
147 | |
148 NavigationEvent* nav_event = FindNavigationEvent(target_url, target_tab_id); | |
149 if (!nav_event) { | |
150 // We cannot find a single navigation event related to this download. | |
151 return NAVIGATION_EVENT_NOT_FOUND; | |
152 } | |
153 | |
154 AddToReferrerChain(out_referrer_chain, nav_event, | |
155 ReferrerChainEntry::DOWNLOAD_URL); | |
156 AttributionResult result = SUCCESS; | |
157 int user_gesture_count = 0; | |
158 while (user_gesture_count < user_gesture_count_limit) { | |
159 // Back trace to the next nav_event that was initiated by the user. | |
160 while (!nav_event->is_user_initiated) { | |
161 nav_event = | |
162 FindNavigationEvent(nav_event->source_url, nav_event->source_tab_id); | |
163 if (!nav_event) | |
164 return result; | |
165 AddToReferrerChain(out_referrer_chain, nav_event, | |
166 nav_event->has_server_redirect | |
167 ? ReferrerChainEntry::SERVER_REDIRECT | |
168 : ReferrerChainEntry::CLIENT_REDIRECT); | |
169 } | |
170 | |
171 user_gesture_count++; | |
172 | |
173 // If the source_url and source_main_frame_url of current navigation event | |
174 // are empty, and is_user_initiated is true, this is a browser initiated | |
175 // navigation (e.g. trigged by typing in address bar, clicking on bookmark, | |
176 // etc). We reached the end of the referrer chain. | |
177 if (nav_event->source_url.is_empty() && | |
178 nav_event->source_main_frame_url.is_empty()) { | |
179 DCHECK(nav_event->is_user_initiated); | |
180 return result; | |
181 } | |
182 | |
183 nav_event = | |
184 FindNavigationEvent(nav_event->source_url, nav_event->source_tab_id); | |
185 if (!nav_event) | |
186 return result; | |
187 | |
188 // Landing page of a download refers to the page user directly interacts | |
189 // with to trigger this download (e.g. clicking on download button). Landing | |
190 // referrer page is the one user interacts with right before navigating to | |
191 // the landing page. | |
192 // Since we are tracing navigations backwards, if we've encountered 1 user | |
193 // gesture before this navigaiton event, this is a navigation lead to the | |
Charlie Reis
2016/12/14 07:43:58
nit: navigation
nit: leading
Jialiu Lin
2016/12/14 19:06:29
done. Thanks.
| |
194 // landing page. If we've encountered 2 user gestures, it leads to landing | |
195 // referrer page. | |
196 if (user_gesture_count == 1) { | |
197 AddToReferrerChain(out_referrer_chain, nav_event, | |
198 ReferrerChainEntry::LANDING_PAGE); | |
199 result = SUCCESS_LANDING_PAGE; | |
200 } else if (user_gesture_count == 2) { | |
201 AddToReferrerChain(out_referrer_chain, nav_event, | |
202 ReferrerChainEntry::LANDING_REFERRER); | |
203 result = SUCCESS_LANDING_REFERRER; | |
204 } else { | |
205 NOTREACHED(); | |
206 } | |
207 } | |
208 return result; | |
209 } | |
210 | |
211 void SafeBrowsingNavigationObserverManager:: | |
212 AddReferrerChainToClientDownloadRequest( | |
213 const GURL& download_url, | |
214 content::WebContents* source_contents, | |
215 ClientDownloadRequest* out_request) { | |
216 int download_tab_id = SessionTabHelper::IdForTab(source_contents); | |
217 UMA_HISTOGRAM_BOOLEAN( | |
218 "SafeBrowsing.ReferrerHasInvalidTabID.DownloadAttribution", | |
219 download_tab_id == -1); | |
220 std::vector<ReferrerChainEntry> attribution_chain; | |
221 AttributionResult result = IdentifyReferrerChain( | |
222 download_url, download_tab_id, kDownloadAttributionUserGestureLimit, | |
223 &attribution_chain); | |
224 UMA_HISTOGRAM_COUNTS_100( | |
225 "SafeBrowsing.ReferrerURLChainSize.DownloadAttribution", | |
226 attribution_chain.size()); | |
227 UMA_HISTOGRAM_ENUMERATION( | |
228 "SafeBrowsing.ReferrerAttributionResult.DownloadAttribution", result, | |
229 SafeBrowsingNavigationObserverManager::ATTRIBUTION_FAILURE_TYPE_MAX); | |
230 for (auto entry : attribution_chain) | |
231 *out_request->add_referrer_chain() = entry; | |
108 } | 232 } |
109 | 233 |
110 SafeBrowsingNavigationObserverManager:: | 234 SafeBrowsingNavigationObserverManager:: |
111 ~SafeBrowsingNavigationObserverManager() {} | 235 ~SafeBrowsingNavigationObserverManager() {} |
112 | 236 |
113 void SafeBrowsingNavigationObserverManager::Observe( | 237 void SafeBrowsingNavigationObserverManager::Observe( |
114 int type, | 238 int type, |
115 const content::NotificationSource& source, | 239 const content::NotificationSource& source, |
116 const content::NotificationDetails& details) { | 240 const content::NotificationDetails& details) { |
117 if (type == chrome::NOTIFICATION_RETARGETING) | 241 if (type == chrome::NOTIFICATION_RETARGETING) |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
159 OnUserGestureConsumed(it->first, it->second); | 283 OnUserGestureConsumed(it->first, it->second); |
160 } else { | 284 } else { |
161 nav_event.is_user_initiated = false; | 285 nav_event.is_user_initiated = false; |
162 } | 286 } |
163 | 287 |
164 auto insertion_result = navigation_map_.insert( | 288 auto insertion_result = navigation_map_.insert( |
165 std::make_pair(target_url, std::vector<NavigationEvent>())); | 289 std::make_pair(target_url, std::vector<NavigationEvent>())); |
166 insertion_result.first->second.push_back(std::move(nav_event)); | 290 insertion_result.first->second.push_back(std::move(nav_event)); |
167 } | 291 } |
168 | 292 |
293 void SafeBrowsingNavigationObserverManager::CleanUpNavigationEvents() { | |
294 // Remove any stale NavigationEnvent, if it is older than | |
295 // kNavigationFootprintTTLInSecond. | |
296 for (auto it = navigation_map_.begin(); it != navigation_map_.end();) { | |
297 it->second.erase(std::remove_if(it->second.begin(), it->second.end(), | |
298 [](const NavigationEvent& nav_event) { | |
299 return IsEventExpired( | |
300 nav_event.last_updated, | |
301 kNavigationFootprintTTLInSecond); | |
302 }), | |
303 it->second.end()); | |
304 if (it->second.size() == 0) | |
305 it = navigation_map_.erase(it); | |
306 else | |
307 ++it; | |
308 } | |
309 } | |
310 | |
311 void SafeBrowsingNavigationObserverManager::CleanUpUserGestures() { | |
312 for (auto it = user_gesture_map_.begin(); it != user_gesture_map_.end();) { | |
313 if (IsEventExpired(it->second, kUserGestureTTLInSecond)) | |
314 it = user_gesture_map_.erase(it); | |
315 else | |
316 ++it; | |
317 } | |
318 } | |
319 | |
320 void SafeBrowsingNavigationObserverManager::CleanUpIpAddresses() { | |
321 for (auto it = host_to_ip_map_.begin(); it != host_to_ip_map_.end();) { | |
322 it->second.erase(std::remove_if(it->second.begin(), it->second.end(), | |
323 [](const ResolvedIPAddress& resolved_ip) { | |
324 return IsEventExpired( | |
325 resolved_ip.timestamp, | |
326 kNavigationFootprintTTLInSecond); | |
327 }), | |
328 it->second.end()); | |
329 if (it->second.size() == 0) | |
330 it = host_to_ip_map_.erase(it); | |
331 else | |
332 ++it; | |
333 } | |
334 } | |
335 | |
336 bool SafeBrowsingNavigationObserverManager::IsCleanUpScheduled() const { | |
337 return cleanup_timer_.IsRunning(); | |
338 } | |
339 | |
340 void SafeBrowsingNavigationObserverManager::ScheduleNextCleanUpAfterInterval( | |
341 base::TimeDelta interval) { | |
342 DCHECK_GT(interval, base::TimeDelta()); | |
343 cleanup_timer_.Stop(); | |
344 cleanup_timer_.Start( | |
345 FROM_HERE, interval, this, | |
346 &SafeBrowsingNavigationObserverManager::CleanUpStaleNavigationFootprints); | |
347 } | |
348 | |
349 NavigationEvent* SafeBrowsingNavigationObserverManager::FindNavigationEvent( | |
350 const GURL& target_url, | |
351 int target_tab_id) { | |
352 auto it = navigation_map_.find(target_url); | |
353 if (it == navigation_map_.end()) { | |
354 return nullptr; | |
355 } | |
356 // Since navigation events are recorded in chronological order, we traverse | |
357 // the vector in reverse order to get the latest match. | |
358 for (auto rit = it->second.rbegin(); rit != it->second.rend(); ++rit) { | |
359 // If tab id is not valid, we only compare url, otherwise we compare both. | |
360 if (rit->destination_url == target_url && | |
361 (target_tab_id == -1 || rit->target_tab_id == target_tab_id)) { | |
362 // If both source_url and source_main_frame_url are empty, and this | |
363 // navigation is not triggered by user, a retargeting navigation probably | |
364 // causes this navigation. In this case, we skip this navigation event and | |
365 // looks for the retargeting navigation event. | |
366 if (rit->source_url.is_empty() && rit->source_main_frame_url.is_empty() && | |
367 !rit->is_user_initiated) { | |
368 continue; | |
369 } else { | |
370 return &*rit; | |
371 } | |
372 } | |
373 } | |
374 return nullptr; | |
375 } | |
376 | |
377 void SafeBrowsingNavigationObserverManager::AddToReferrerChain( | |
378 std::vector<ReferrerChainEntry>* referrer_chain, | |
379 NavigationEvent* nav_event, | |
380 ReferrerChainEntry::URLType type) { | |
381 ReferrerChainEntry referrer_chain_entry; | |
382 referrer_chain_entry.set_url(nav_event->destination_url.spec()); | |
383 referrer_chain_entry.set_type(type); | |
384 auto ip_it = host_to_ip_map_.find(nav_event->destination_url.host()); | |
385 if (ip_it != host_to_ip_map_.end()) { | |
386 for (ResolvedIPAddress entry : ip_it->second) { | |
387 referrer_chain_entry.add_ip_addresses(entry.ip); | |
388 } | |
389 } | |
390 // Since we only track navigation to landing referrer, we will not log the | |
391 // referrer of the landing referrer page. | |
392 if (type != ReferrerChainEntry::LANDING_REFERRER) { | |
393 referrer_chain_entry.set_referrer_url(nav_event->source_url.spec()); | |
394 referrer_chain_entry.set_referrer_main_frame_url( | |
395 nav_event->source_main_frame_url.spec()); | |
396 } | |
397 referrer_chain_entry.set_is_retargeting(nav_event->source_tab_id != | |
398 nav_event->target_tab_id); | |
399 referrer_chain_entry.set_navigation_time_msec( | |
400 nav_event->last_updated.ToJavaTime()); | |
401 referrer_chain->push_back(referrer_chain_entry); | |
402 } | |
403 | |
169 } // namespace safe_browsing | 404 } // namespace safe_browsing |
OLD | NEW |