OLD | NEW |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager
.h" | 5 #include "chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager
.h" |
6 | 6 |
7 #include "base/memory/ptr_util.h" | 7 #include "base/memory/ptr_util.h" |
| 8 #include "base/metrics/histogram_macros.h" |
| 9 #include "base/strings/stringprintf.h" |
8 #include "base/time/time.h" | 10 #include "base/time/time.h" |
| 11 #include "base/timer/timer.h" |
9 #include "chrome/browser/chrome_notification_types.h" | 12 #include "chrome/browser/chrome_notification_types.h" |
10 #include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h" | 13 #include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h" |
11 #include "chrome/browser/sessions/session_tab_helper.h" | 14 #include "chrome/browser/sessions/session_tab_helper.h" |
12 #include "chrome/browser/tab_contents/retargeting_details.h" | 15 #include "chrome/browser/tab_contents/retargeting_details.h" |
13 #include "content/public/browser/navigation_details.h" | 16 #include "content/public/browser/navigation_details.h" |
14 #include "content/public/browser/notification_service.h" | 17 #include "content/public/browser/notification_service.h" |
15 #include "content/public/browser/notification_types.h" | 18 #include "content/public/browser/notification_types.h" |
16 #include "content/public/browser/render_frame_host.h" | 19 #include "content/public/browser/render_frame_host.h" |
17 #include "content/public/browser/render_process_host.h" | 20 #include "content/public/browser/render_process_host.h" |
18 #include "content/public/browser/web_contents.h" | 21 #include "content/public/browser/web_contents.h" |
19 | 22 |
20 using content::WebContents; | 23 using content::WebContents; |
21 | 24 |
22 namespace safe_browsing { | 25 namespace safe_browsing { |
23 | 26 |
| 27 namespace { |
| 28 |
| 29 // Given when an event happened and its TTL, determine if it is already expired. |
| 30 // Note, if for some reason this event's timestamp is in the future, this |
| 31 // event's timestamp is invalid, hence we treat it as expired. |
| 32 bool IsEventExpired(const base::Time& event_time, double ttl_in_second) { |
| 33 double current_time_in_second = base::Time::Now().ToDoubleT(); |
| 34 double event_time_in_second = event_time.ToDoubleT(); |
| 35 if (current_time_in_second <= event_time_in_second) |
| 36 return true; |
| 37 return current_time_in_second - event_time_in_second > ttl_in_second; |
| 38 } |
| 39 |
| 40 } // namespace |
| 41 |
24 // The expiration period of a user gesture. Any user gesture that happened 1.0 | 42 // The expiration period of a user gesture. Any user gesture that happened 1.0 |
25 // second ago will be considered as expired and not relevant to upcoming | 43 // second ago is considered as expired and not relevant to upcoming navigation |
26 // navigation events. | 44 // events. |
27 static const double kUserGestureTTLInSecond = 1.0; | 45 static const double kUserGestureTTLInSecond = 1.0; |
| 46 // The expiration period of navigation events and resolved IP addresses. Any |
| 47 // navigation related records that happened 2 minutes ago are considered as |
| 48 // expired. So we clean up these navigation footprints every 2 minutes. |
| 49 static const double kNavigationFootprintTTLInSecond = 120.0; |
| 50 // The number of user gestures we trace back for download attribution. |
| 51 static const int kDownloadAttributionUserGestureLimit = 2; |
28 | 52 |
29 // static | 53 // static |
30 bool SafeBrowsingNavigationObserverManager::IsUserGestureExpired( | 54 bool SafeBrowsingNavigationObserverManager::IsUserGestureExpired( |
31 const base::Time& timestamp) { | 55 const base::Time& timestamp) { |
32 double now = base::Time::Now().ToDoubleT(); | 56 return IsEventExpired(timestamp, kUserGestureTTLInSecond); |
33 double timestamp_in_double = timestamp.ToDoubleT(); | |
34 | |
35 if (now <= timestamp_in_double) | |
36 return true; | |
37 return (now - timestamp_in_double) > kUserGestureTTLInSecond; | |
38 } | 57 } |
39 | 58 |
40 // static | 59 // static |
41 GURL SafeBrowsingNavigationObserverManager::ClearEmptyRef(const GURL& url) { | 60 GURL SafeBrowsingNavigationObserverManager::ClearEmptyRef(const GURL& url) { |
42 if (url.has_ref() && url.ref().empty()) { | 61 if (url.has_ref() && url.ref().empty()) { |
43 url::Replacements<char> replacements; | 62 url::Replacements<char> replacements; |
44 replacements.ClearRef(); | 63 replacements.ClearRef(); |
45 return url.ReplaceComponents(replacements); | 64 return url.ReplaceComponents(replacements); |
46 } | 65 } |
47 return url; | 66 return url; |
48 } | 67 } |
49 | 68 |
50 SafeBrowsingNavigationObserverManager::SafeBrowsingNavigationObserverManager() { | 69 SafeBrowsingNavigationObserverManager::SafeBrowsingNavigationObserverManager() { |
51 registrar_.Add(this, chrome::NOTIFICATION_RETARGETING, | 70 registrar_.Add(this, chrome::NOTIFICATION_RETARGETING, |
52 content::NotificationService::AllSources()); | 71 content::NotificationService::AllSources()); |
| 72 |
| 73 // TODO(jialiul): call ScheduleNextCleanUpAfterInterval() when this class is |
| 74 // ready to be hooked into SafeBrowsingService. |
53 } | 75 } |
54 | 76 |
55 void SafeBrowsingNavigationObserverManager::RecordNavigationEvent( | 77 void SafeBrowsingNavigationObserverManager::RecordNavigationEvent( |
56 const GURL& nav_event_key, | 78 const GURL& nav_event_key, |
57 NavigationEvent* nav_event) { | 79 NavigationEvent* nav_event) { |
58 auto insertion_result = navigation_map_.insert( | 80 auto insertion_result = navigation_map_.insert( |
59 std::make_pair(nav_event_key, std::vector<NavigationEvent>())); | 81 std::make_pair(nav_event_key, std::vector<NavigationEvent>())); |
60 | 82 |
61 insertion_result.first->second.push_back(std::move(*nav_event)); | 83 insertion_result.first->second.push_back(std::move(*nav_event)); |
62 } | 84 } |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
97 } | 119 } |
98 } | 120 } |
99 // If this is a new IP of this host, and we added to the end of the vector. | 121 // If this is a new IP of this host, and we added to the end of the vector. |
100 insert_result.first->second.push_back( | 122 insert_result.first->second.push_back( |
101 ResolvedIPAddress(base::Time::Now(), ip)); | 123 ResolvedIPAddress(base::Time::Now(), ip)); |
102 } | 124 } |
103 | 125 |
104 void SafeBrowsingNavigationObserverManager::OnWebContentDestroyed( | 126 void SafeBrowsingNavigationObserverManager::OnWebContentDestroyed( |
105 content::WebContents* web_contents) { | 127 content::WebContents* web_contents) { |
106 user_gesture_map_.erase(web_contents); | 128 user_gesture_map_.erase(web_contents); |
107 // TODO (jialiul): Will add other clean up tasks shortly. | 129 } |
| 130 |
| 131 void SafeBrowsingNavigationObserverManager::CleanUpStaleNavigationFootprints() { |
| 132 CleanUpNavigationEvents(); |
| 133 CleanUpUserGestures(); |
| 134 CleanUpIpAddresses(); |
| 135 ScheduleNextCleanUpAfterInterval( |
| 136 base::TimeDelta::FromSecondsD(kNavigationFootprintTTLInSecond)); |
| 137 } |
| 138 |
| 139 SafeBrowsingNavigationObserverManager::AttributionResult |
| 140 SafeBrowsingNavigationObserverManager::IdentifyReferrerChain( |
| 141 const GURL& target_url, |
| 142 int target_tab_id, |
| 143 int user_gesture_count_limit, |
| 144 std::vector<ReferrerChainEntry>* out_referrer_chain) { |
| 145 if (!target_url.is_valid()) |
| 146 return INVALID_URL; |
| 147 |
| 148 NavigationEvent* nav_event = FindNavigationEvent(target_url, target_tab_id); |
| 149 if (!nav_event) { |
| 150 // We cannot find a single navigation event related to this download. |
| 151 return NAVIGATION_EVENT_NOT_FOUND; |
| 152 } |
| 153 |
| 154 AddToReferrerChain(out_referrer_chain, nav_event, |
| 155 ReferrerChainEntry::DOWNLOAD_URL); |
| 156 AttributionResult result = SUCCESS; |
| 157 int user_gesture_count = 0; |
| 158 while (user_gesture_count < user_gesture_count_limit) { |
| 159 // Back trace to the next nav_event that was initiated by the user. |
| 160 while (!nav_event->is_user_initiated) { |
| 161 nav_event = |
| 162 FindNavigationEvent(nav_event->source_url, nav_event->source_tab_id); |
| 163 if (!nav_event) |
| 164 return result; |
| 165 AddToReferrerChain(out_referrer_chain, nav_event, |
| 166 nav_event->has_server_redirect |
| 167 ? ReferrerChainEntry::SERVER_REDIRECT |
| 168 : ReferrerChainEntry::CLIENT_REDIRECT); |
| 169 } |
| 170 |
| 171 user_gesture_count++; |
| 172 |
| 173 // If the source_url and source_main_frame_url of current navigation event |
| 174 // are empty, and is_user_initiated is true, this is a browser initiated |
| 175 // navigation (e.g. trigged by typing in address bar, clicking on bookmark, |
| 176 // etc). We reached the end of the referrer chain. |
| 177 if (nav_event->source_url.is_empty() && |
| 178 nav_event->source_main_frame_url.is_empty()) { |
| 179 DCHECK(nav_event->is_user_initiated); |
| 180 return result; |
| 181 } |
| 182 |
| 183 nav_event = |
| 184 FindNavigationEvent(nav_event->source_url, nav_event->source_tab_id); |
| 185 if (!nav_event) |
| 186 return result; |
| 187 |
| 188 // Landing page of a download refers to the page user directly interacts |
| 189 // with to trigger this download (e.g. clicking on download button). Landing |
| 190 // referrer page is the one user interacts with right before navigating to |
| 191 // the landing page. |
| 192 // Since we are tracing navigations backwards, if we've encountered 1 user |
| 193 // gesture before this navigation event, this is a navigation leading to the |
| 194 // landing page. If we've encountered 2 user gestures, it leads to landing |
| 195 // referrer page. |
| 196 if (user_gesture_count == 1) { |
| 197 AddToReferrerChain(out_referrer_chain, nav_event, |
| 198 ReferrerChainEntry::LANDING_PAGE); |
| 199 result = SUCCESS_LANDING_PAGE; |
| 200 } else if (user_gesture_count == 2) { |
| 201 AddToReferrerChain(out_referrer_chain, nav_event, |
| 202 ReferrerChainEntry::LANDING_REFERRER); |
| 203 result = SUCCESS_LANDING_REFERRER; |
| 204 } else { |
| 205 NOTREACHED(); |
| 206 } |
| 207 } |
| 208 return result; |
| 209 } |
| 210 |
| 211 void SafeBrowsingNavigationObserverManager:: |
| 212 AddReferrerChainToClientDownloadRequest( |
| 213 const GURL& download_url, |
| 214 content::WebContents* source_contents, |
| 215 ClientDownloadRequest* out_request) { |
| 216 int download_tab_id = SessionTabHelper::IdForTab(source_contents); |
| 217 UMA_HISTOGRAM_BOOLEAN( |
| 218 "SafeBrowsing.ReferrerHasInvalidTabID.DownloadAttribution", |
| 219 download_tab_id == -1); |
| 220 std::vector<ReferrerChainEntry> attribution_chain; |
| 221 AttributionResult result = IdentifyReferrerChain( |
| 222 download_url, download_tab_id, kDownloadAttributionUserGestureLimit, |
| 223 &attribution_chain); |
| 224 UMA_HISTOGRAM_COUNTS_100( |
| 225 "SafeBrowsing.ReferrerURLChainSize.DownloadAttribution", |
| 226 attribution_chain.size()); |
| 227 UMA_HISTOGRAM_ENUMERATION( |
| 228 "SafeBrowsing.ReferrerAttributionResult.DownloadAttribution", result, |
| 229 SafeBrowsingNavigationObserverManager::ATTRIBUTION_FAILURE_TYPE_MAX); |
| 230 for (auto entry : attribution_chain) |
| 231 *out_request->add_referrer_chain() = entry; |
108 } | 232 } |
109 | 233 |
110 SafeBrowsingNavigationObserverManager:: | 234 SafeBrowsingNavigationObserverManager:: |
111 ~SafeBrowsingNavigationObserverManager() {} | 235 ~SafeBrowsingNavigationObserverManager() {} |
112 | 236 |
113 void SafeBrowsingNavigationObserverManager::Observe( | 237 void SafeBrowsingNavigationObserverManager::Observe( |
114 int type, | 238 int type, |
115 const content::NotificationSource& source, | 239 const content::NotificationSource& source, |
116 const content::NotificationDetails& details) { | 240 const content::NotificationDetails& details) { |
117 if (type == chrome::NOTIFICATION_RETARGETING) | 241 if (type == chrome::NOTIFICATION_RETARGETING) |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
159 OnUserGestureConsumed(it->first, it->second); | 283 OnUserGestureConsumed(it->first, it->second); |
160 } else { | 284 } else { |
161 nav_event.is_user_initiated = false; | 285 nav_event.is_user_initiated = false; |
162 } | 286 } |
163 | 287 |
164 auto insertion_result = navigation_map_.insert( | 288 auto insertion_result = navigation_map_.insert( |
165 std::make_pair(target_url, std::vector<NavigationEvent>())); | 289 std::make_pair(target_url, std::vector<NavigationEvent>())); |
166 insertion_result.first->second.push_back(std::move(nav_event)); | 290 insertion_result.first->second.push_back(std::move(nav_event)); |
167 } | 291 } |
168 | 292 |
| 293 void SafeBrowsingNavigationObserverManager::CleanUpNavigationEvents() { |
| 294 // Remove any stale NavigationEnvent, if it is older than |
| 295 // kNavigationFootprintTTLInSecond. |
| 296 for (auto it = navigation_map_.begin(); it != navigation_map_.end();) { |
| 297 it->second.erase(std::remove_if(it->second.begin(), it->second.end(), |
| 298 [](const NavigationEvent& nav_event) { |
| 299 return IsEventExpired( |
| 300 nav_event.last_updated, |
| 301 kNavigationFootprintTTLInSecond); |
| 302 }), |
| 303 it->second.end()); |
| 304 if (it->second.size() == 0) |
| 305 it = navigation_map_.erase(it); |
| 306 else |
| 307 ++it; |
| 308 } |
| 309 } |
| 310 |
| 311 void SafeBrowsingNavigationObserverManager::CleanUpUserGestures() { |
| 312 for (auto it = user_gesture_map_.begin(); it != user_gesture_map_.end();) { |
| 313 if (IsEventExpired(it->second, kUserGestureTTLInSecond)) |
| 314 it = user_gesture_map_.erase(it); |
| 315 else |
| 316 ++it; |
| 317 } |
| 318 } |
| 319 |
| 320 void SafeBrowsingNavigationObserverManager::CleanUpIpAddresses() { |
| 321 for (auto it = host_to_ip_map_.begin(); it != host_to_ip_map_.end();) { |
| 322 it->second.erase(std::remove_if(it->second.begin(), it->second.end(), |
| 323 [](const ResolvedIPAddress& resolved_ip) { |
| 324 return IsEventExpired( |
| 325 resolved_ip.timestamp, |
| 326 kNavigationFootprintTTLInSecond); |
| 327 }), |
| 328 it->second.end()); |
| 329 if (it->second.size() == 0) |
| 330 it = host_to_ip_map_.erase(it); |
| 331 else |
| 332 ++it; |
| 333 } |
| 334 } |
| 335 |
| 336 bool SafeBrowsingNavigationObserverManager::IsCleanUpScheduled() const { |
| 337 return cleanup_timer_.IsRunning(); |
| 338 } |
| 339 |
| 340 void SafeBrowsingNavigationObserverManager::ScheduleNextCleanUpAfterInterval( |
| 341 base::TimeDelta interval) { |
| 342 DCHECK_GT(interval, base::TimeDelta()); |
| 343 cleanup_timer_.Stop(); |
| 344 cleanup_timer_.Start( |
| 345 FROM_HERE, interval, this, |
| 346 &SafeBrowsingNavigationObserverManager::CleanUpStaleNavigationFootprints); |
| 347 } |
| 348 |
| 349 NavigationEvent* SafeBrowsingNavigationObserverManager::FindNavigationEvent( |
| 350 const GURL& target_url, |
| 351 int target_tab_id) { |
| 352 auto it = navigation_map_.find(target_url); |
| 353 if (it == navigation_map_.end()) { |
| 354 return nullptr; |
| 355 } |
| 356 // Since navigation events are recorded in chronological order, we traverse |
| 357 // the vector in reverse order to get the latest match. |
| 358 for (auto rit = it->second.rbegin(); rit != it->second.rend(); ++rit) { |
| 359 // If tab id is not valid, we only compare url, otherwise we compare both. |
| 360 if (rit->destination_url == target_url && |
| 361 (target_tab_id == -1 || rit->target_tab_id == target_tab_id)) { |
| 362 // If both source_url and source_main_frame_url are empty, and this |
| 363 // navigation is not triggered by user, a retargeting navigation probably |
| 364 // causes this navigation. In this case, we skip this navigation event and |
| 365 // looks for the retargeting navigation event. |
| 366 if (rit->source_url.is_empty() && rit->source_main_frame_url.is_empty() && |
| 367 !rit->is_user_initiated) { |
| 368 continue; |
| 369 } else { |
| 370 return &*rit; |
| 371 } |
| 372 } |
| 373 } |
| 374 return nullptr; |
| 375 } |
| 376 |
| 377 void SafeBrowsingNavigationObserverManager::AddToReferrerChain( |
| 378 std::vector<ReferrerChainEntry>* referrer_chain, |
| 379 NavigationEvent* nav_event, |
| 380 ReferrerChainEntry::URLType type) { |
| 381 ReferrerChainEntry referrer_chain_entry; |
| 382 referrer_chain_entry.set_url(nav_event->destination_url.spec()); |
| 383 referrer_chain_entry.set_type(type); |
| 384 auto ip_it = host_to_ip_map_.find(nav_event->destination_url.host()); |
| 385 if (ip_it != host_to_ip_map_.end()) { |
| 386 for (ResolvedIPAddress entry : ip_it->second) { |
| 387 referrer_chain_entry.add_ip_addresses(entry.ip); |
| 388 } |
| 389 } |
| 390 // Since we only track navigation to landing referrer, we will not log the |
| 391 // referrer of the landing referrer page. |
| 392 if (type != ReferrerChainEntry::LANDING_REFERRER) { |
| 393 referrer_chain_entry.set_referrer_url(nav_event->source_url.spec()); |
| 394 referrer_chain_entry.set_referrer_main_frame_url( |
| 395 nav_event->source_main_frame_url.spec()); |
| 396 } |
| 397 referrer_chain_entry.set_is_retargeting(nav_event->source_tab_id != |
| 398 nav_event->target_tab_id); |
| 399 referrer_chain_entry.set_navigation_time_msec( |
| 400 nav_event->last_updated.ToJavaTime()); |
| 401 referrer_chain->push_back(referrer_chain_entry); |
| 402 } |
| 403 |
169 } // namespace safe_browsing | 404 } // namespace safe_browsing |
OLD | NEW |