Add SafeBrowsingNavigationObserver to listen to navigation events

chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h"
+
+#include "base/memory/ptr_util.h"
+#include "base/time/time.h"
+#include "chrome/browser/chrome_notification_types.h"
+#include "chrome/browser/sessions/session_tab_helper.h"
+#include "chrome/browser/tab_contents/retargeting_details.h"
+#include "content/public/browser/navigation_details.h"
+#include "content/public/browser/navigation_handle.h"
+#include "content/public/browser/notification_service.h"
+#include "content/public/browser/notification_types.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/render_process_host.h"
+#include "content/public/browser/resource_request_details.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/common/resource_type.h"
+
+using content::WebContents;
+namespace safe_browsing {
+
+static const double kUserGestureTTLInSecond = 0.5;
+
+bool IsExpired(const base::Time& timestamp) {

[Charlie Reis, 2016/09/23 23:14:13]

nit: IsUserGestureExpired?

[Jialiu Lin, 2016/09/27 17:50:59]

Done.

+  double now = base::Time::Now().ToDoubleT();
+  double timestamp_in_double = timestamp.ToDoubleT();
+  return now > timestamp_in_double
+             ? now - timestamp_in_double > kUserGestureTTLInSecond
+             : true;
+}
+
+// SafeBrowsingNavigationObserver::NavigationEvent-----------------------------
+NavigationEvent::NavigationEvent()
+    : source_url(),
+      source_tab_id(-1),
+      target_url(),
+      target_tab_id(-1),
+      frame_id(-1),
+      main_frame_url(),
+      timestamp(base::Time::Now()),
+      is_user_initiated(false),
+      has_committed(false),
+      is_server_redirect(false),
+      server_redirect_url() {}
+
+NavigationEvent::NavigationEvent(NavigationEvent&& nav_event)
+    : source_url(std::move(nav_event.source_url)),
+      source_tab_id(std::move(nav_event.source_tab_id)),
+      target_url(std::move(nav_event.target_url)),
+      target_tab_id(std::move(nav_event.target_tab_id)),
+      frame_id(nav_event.frame_id),
+      main_frame_url(std::move(nav_event.main_frame_url)),
+      timestamp(nav_event.timestamp),
+      is_user_initiated(nav_event.is_user_initiated),
+      has_committed(nav_event.has_committed),
+      is_server_redirect(nav_event.is_server_redirect),
+      server_redirect_url(std::move(nav_event.server_redirect_url)) {}
+
+NavigationEvent& NavigationEvent::operator=(NavigationEvent&& nav_event) {
+  source_url = std::move(nav_event.source_url);
+  source_tab_id = std::move(nav_event.source_tab_id);
+  target_url = std::move(nav_event.target_url);
+  target_tab_id = std::move(nav_event.target_tab_id);
+  frame_id = nav_event.frame_id;
+  main_frame_url = std::move(nav_event.main_frame_url);
+  timestamp = nav_event.timestamp;
+  is_user_initiated = nav_event.is_user_initiated;
+  has_committed = nav_event.has_committed;
+  is_server_redirect = nav_event.is_server_redirect;
+  server_redirect_url = std::move(nav_event.server_redirect_url);
+  return *this;
+}
+
+NavigationEvent::~NavigationEvent() {}
+
+// -------------------SBNavigationObserverManager Functions----------------
+
+SBNavigationObserverManager::SBNavigationObserverManager() {
+  registrar_.Add(this, chrome::NOTIFICATION_RETARGETING,
+                 content::NotificationService::AllSources());
+}
+
+void SBNavigationObserverManager::RecordNavigationEvent(
+    const GURL& nav_event_key,
+    NavigationEvent* nav_event) {
+  auto insertion_result = navigation_map_.insert(
+      std::make_pair(nav_event_key, std::vector<NavigationEvent>()));

[Charlie Reis, 2016/09/23 23:14:14]

Maybe I'm misreading this, but isn't this only needed if there's no vector
already?  If there's a pre-existing vector for this key, we presumably just want
to push the event into it and not insert a new empty vector.

[Jialiu Lin, 2016/09/27 17:50:59]

http://en.cppreference.com/w/cpp/container/unordered_map/insert
If the key is already there, insertion will not happen.
std::unordered_map::insert returns a pair<iterator, bool>, this bool indicates
if insertion actually happened, the iterator point to the latest inserted entry
or the one prevent the insertion.

[Charlie Reis, 2016/09/28 21:08:13]

Acknowledged.

+
+  insertion_result.first->second.push_back(std::move(*nav_event));
+}
+
+void SBNavigationObserverManager::RecordUserGestureForWebContents(
+    content::WebContents* web_contents,
+    const base::Time& timestamp) {
+  auto insertion_result =
+      user_gesture_map_.insert(std::make_pair(web_contents, timestamp));
+  if (!insertion_result.second)
+    insertion_result.first->second = timestamp;
+}
+
+void SBNavigationObserverManager::OnUserGestureConsumed(
+    content::WebContents* web_contents,
+    const base::Time& timestamp) {
+  auto it = user_gesture_map_.find(web_contents);
+  if (it == user_gesture_map_.end())
+    return;
+  if (timestamp >= it->second)
+    user_gesture_map_.erase(web_contents);
+}
+
+void SBNavigationObserverManager::OnWebContentDestroyed(
+    content::WebContents* web_contents) {
+  user_gesture_map_.erase(web_contents);
+  // TODO (jialiul): Will add other clean up tasks shortly.
+}
+
+SBNavigationObserverManager::~SBNavigationObserverManager(){}
+
+void SBNavigationObserverManager::Observe(
+    int type,
+    const content::NotificationSource& source,
+    const content::NotificationDetails& details) {
+  if (type == chrome::NOTIFICATION_RETARGETING)
+    RecordRetargeting(details);
+}
+
+void SBNavigationObserverManager::RecordRetargeting(
+    const content::NotificationDetails& details) {
+  const RetargetingDetails* retargeting_detail =
+      content::Details<const RetargetingDetails>(details).ptr();
+  DCHECK(retargeting_detail);
+  content::WebContents* source_contents =
+      retargeting_detail->source_web_contents;
+  content::WebContents* target_contents =
+      retargeting_detail->target_web_contents;
+  DCHECK(source_contents);
+  DCHECK(target_contents);
+
+  content::RenderFrameHost* rfh = content::RenderFrameHost::FromID(
+      source_contents->GetRenderProcessHost()->GetID(),

[Charlie Reis, 2016/09/23 23:14:13]

This would break with OOPIFs, since WebContents::GetRenderProcessHost() is for
the main frame and might not match the RFH's process ID.  I filed
https://crbug.com/649855, since RetargetingDetails apparently lacks the frame's
process ID to be able to get this right.

[Jialiu Lin, 2016/09/27 17:50:59]

Thanks, I will add a comments here. 

+      retargeting_detail->source_render_frame_id);
+  GURL::Replacements replacements;
+  replacements.ClearRef();

[Charlie Reis, 2016/09/23 23:14:13]

Can you help me understand why we're clearing the ref everywhere?  I would
suggest a helper function for it, but I'm not sure if we want to do it at all.

For example, if it's so that in-page navigations look the same as the original
page load, that will only help with fragment navigations and not pushState or
replaceState (which can change the URL to any path within the same origin).

[Jialiu Lin, 2016/09/27 17:50:59]

yes, this is to make "http://foo.com#bar" treated the same as "http://foo.com". 
I added a helper function ClearURLRef(..) for it. 

I don't quite understand the pushState or replaceState part. Can you maybe give
me an example?

[Charlie Reis, 2016/09/28 21:08:13]

Sure.  For both #bar and pushState/replaceState, we're talking about in-page
navigations that don't have a new document and keep all their existing iframes
and other state.

Clearing the ref may give you a kind of canonical URL for the fragment case, but
it won't help with pushState.  Suppose we're on http://foo.com/pageone.  That
page can do history.pushState({}, "anything", "anything"), at which point the
URL becomes http://foo.com/anything.  There's no way to tell just by looking at
the URL that "pageone" and "anything" are the same document.

This means that clearing the ref is not an effective way to tell if we're still
on the same page or not.  As a result, I'd recommend not doing it, and we can
look for other ways to achieve the goal you're after.

What's the problem you were trying to solve?  (In other words, why did you need
to treat foo.com and foo.com#bar the same?)

[Jialiu Lin, 2016/09/28 21:39:14]

I see.  How about I just remove empty ref (a.k.a
"http://foo.com#"->"http://foo.com")? Since a lot of javascript triggered
navigation does <a href="#" onclick="somefunction()">. Having two keys
"http://foo.com#" and "http://foo.com" in NavigationMap might be a little bit
confusing when I actually need to trace back downloads.

+  const GURL& target_url =
+      retargeting_detail->target_url.ReplaceComponents(replacements);
+
+  navigation_map_.insert(
+      std::make_pair(target_url, std::vector<NavigationEvent>()));
+
+  NavigationEvent nav_event;
+  nav_event.source_url =
+      rfh ? rfh->GetLastCommittedURL().ReplaceComponents(replacements) : GURL();
+  nav_event.source_tab_id = SessionTabHelper::IdForTab(source_contents);
+  nav_event.target_url = target_url;
+  nav_event.target_tab_id = SessionTabHelper::IdForTab(target_contents);
+  nav_event.frame_id = rfh ? rfh->GetFrameTreeNodeId() : -1;
+  nav_event.main_frame_url =
+      source_contents->GetLastCommittedURL().ReplaceComponents(replacements);
+  nav_event.timestamp = base::Time::Now();
+  auto it = user_gesture_map_.find(source_contents);
+  if (it != user_gesture_map_.end() && !IsExpired(it->second)) {
+    nav_event.is_user_initiated = true;
+    OnUserGestureConsumed(it->first, it->second);
+  } else {
+    nav_event.is_user_initiated = false;
+  }
+
+  navigation_map_[target_url].push_back(std::move(nav_event));
+}
+
+// -------------------SafeBrowsingNavigationObserver Functions----------------
+
+// static
+const char SafeBrowsingNavigationObserver::kWebContentsUserDataKey[] =
+    "web_contents_safe_browsing_navigation_observer";
+
+// static
+void SafeBrowsingNavigationObserver::MaybeCreateForWebContents(
+    content::WebContents* web_contents) {
+  if (FromWebContents(web_contents))
+    return;
+  // TODO(jialiul): This function will be called by TabHelpers::AttachTabHelpers
+  // Complete this function when the entire class is ready.
+}
+
+// static
+SafeBrowsingNavigationObserver* SafeBrowsingNavigationObserver::FromWebContents(
+    content::WebContents* web_contents) {
+  return static_cast<SafeBrowsingNavigationObserver*>(
+      web_contents->GetUserData(kWebContentsUserDataKey));
+}
+
+SafeBrowsingNavigationObserver::SafeBrowsingNavigationObserver(
+    content::WebContents* contents,
+    const scoped_refptr<SBNavigationObserverManager>& manager)
+    : content::WebContentsObserver(contents),
+      manager_(manager),
+      has_user_gesture_(false),
+      last_user_gesture_timestamp_(base::Time::Now()) {}
+
+SafeBrowsingNavigationObserver::~SafeBrowsingNavigationObserver() {}
+
+// Called when a navigation started in the WebContents. |navigation_handle| in
+// paramter is unique to this navigation, which will appear in the following
+// DidRedirectNavigation, and DidFinishNavigation call too.
+void SafeBrowsingNavigationObserver::DidStartNavigation(
+    content::NavigationHandle* navigation_handle) {
+  // If we already seen this navigation_handle before, no need to do anything.
+  if (navigation_handle_map_.find(navigation_handle) !=
+      navigation_handle_map_.end())
+    return;
+
+  // Construct a NavigationEvent based on available information in
+  // navigation_handle.
+  NavigationEvent nav_event;
+  content::RenderFrameHost* host =
+      navigation_handle->GetWebContents()->FindFrameByFrameTreeNodeId(
+          navigation_handle->GetFrameTreeNodeId());

[Charlie Reis, 2016/09/23 23:14:13]

I'm concerned about this line, since the navigation could be starting in a
pending RenderFrameHost, not the current RenderFrameHost.  Maybe that doesn't
matter if we're just trying to get the source URL, but I think there's a problem
with that as well (below).

[Jialiu Lin, 2016/09/27 17:50:59]

Agree these are tricky cases. See my proposals below.

+
+  // If there was URL previously committed in this render frame host,
+  // set it as the source url of this navigation. Otherwise, this is the
+  // first url going to commit in this frame. We set navigation_handle's URL as
+  // the source url.
+  GURL::Replacements replacements;
+  replacements.ClearRef();
+  if (host && host->GetLastCommittedURL().is_valid()) {
+    nav_event.source_url =
+        host->GetLastCommittedURL().ReplaceComponents(replacements);

[Charlie Reis, 2016/09/23 23:14:13]

This looks like it won't handle different frames of the same page targeting each
other (which I'm guessing don't go through NOTIFICATION_RETARGETING).  If the
main frame at evil.com creates an iframe at innocent.com and then navigates it
to download.com, source_url will look like innocent.com instead of evil.com.

It's not clear to me whether we have something that tracks the URL of the page
that caused the navigation yet, and it might be something our team will need to
add.  Can you test and verify whether that's correct?

[Jialiu Lin, 2016/09/27 17:50:59]

There are two cases keep me awake at night. 
(1) the first one is just like what you described, i.e. creating a scapegoat
iframe within the same page. 
For this case, at least main frame "evil.com" is captured in
NavigationEvent.main_frame_url, so evil.com will eventually be blacklisted.


(2) evil.com navigates to innocent.com in a new named tab (main frame), then
evil.com tries to download a malicious file in the tab currently displaying
innocent.com. This can be easily done by using target="innocent". The second
navigation will not trigger NOTIFICATION_RETARGETING, so it looks just like
innocent.com is landing a bad download. And to confirm, there is no existing
method (I've known of) can track this case correctly. Though, there are 2 hacky
ways we can try: 
  (a) In SafeBrosingNaivgationObserverManager, create another map that keeps
track of who creates each WebContents (both the creator webcontents and creator
URL). We'll report this creator URL chain (if they are not stale) in download
ping.
  (b)  NavigationHandle::GetReferrer() seems pointing to the correct referrer if
no ReferrerPolicy is enforced. I wonder if it is possible to always keep an
unsanitized referrer, it might involve some upstream changes in navigator and
other classes as well.

[Charlie Reis, 2016/09/28 21:08:13]

Note that my example could have put evil.com in iframe 1 and
innocent.com/download.com in iframe 2.  We would never see evil.com reported,
though you're right that we would at least have whatever main_frame_url is,
which is somewhat indirectly responsible  (e.g., it might be a news web site
that loaded a bad advertisement).
Agreed.  I think that case would be problematic as well.
I'm a little hesitant on this one, again because main frames tend not to be
special.  I worry about cases that the bad URLs are in iframes and we never end
up seeing them.  There's also the risk that the creator of a window is not to
blame, but some other window is able to target it by name.
I'm curious about this approach.  As a strawman proposal, suppose we added a
unsanitized_referrer alongside referrer, such that it's populated even when
referrer has been omitted.  This might let us track what we need in your map. 
The downside is that referrer shows up *everywhere*, so it's potentially a lot
of plumbing.

Otherwise, we may need to bake something into the core navigation logic to track
this kind of thing.  The good news is, our team does see some benefits in going
down that path-- it would help with process model decisions (e.g., which process
should this data URL go into?) would have helped with some recent security bug
fixes.  So maybe it's worth finding a way to track that.

(As a starting point, there's ScheduledNavigation::m_originDocument in Blink,
but Daniel points out that isn't trustworthy or correct in many cases.)

[Jialiu Lin, 2016/09/28 21:39:14]

You're right. it is quite indirect.
Let talk more about these cases. I'll sent out a calendar invite to you, nasko@,
and dcheng@ shortly. 

+  } else {
+    nav_event.source_url =
+        navigation_handle->GetURL().ReplaceComponents(replacements);
+  }
+
+  nav_event.target_url =
+      navigation_handle->GetURL().ReplaceComponents(replacements);
+
+  nav_event.source_tab_id =
+      SessionTabHelper::IdForTab(navigation_handle->GetWebContents());
+  nav_event.timestamp = base::Time::Now();
+  nav_event.frame_id = navigation_handle->GetFrameTreeNodeId();
+
+  if (navigation_handle->IsInMainFrame()) {
+    nav_event.main_frame_url = nav_event.source_url;
+  } else {
+    nav_event.main_frame_url = navigation_handle->GetWebContents()
+                                   ->GetLastCommittedURL()
+                                   .ReplaceComponents(replacements);
+  }
+  if (has_user_gesture_ && !IsExpired(last_user_gesture_timestamp_)) {
+    nav_event.is_user_initiated = has_user_gesture_;
+    manager_->OnUserGestureConsumed(web_contents(),
+                                    last_user_gesture_timestamp_);
+  }
+  has_user_gesture_ = false;
+  navigation_handle_map_.insert(
+      std::make_pair(navigation_handle, std::move(nav_event)));
+}
+
+void SafeBrowsingNavigationObserver::DidRedirectNavigation(
+    content::NavigationHandle* navigation_handle) {
+  // We should have already seen this navigation_handle in DidStartNavigation.
+  if (navigation_handle_map_.find(navigation_handle) ==
+      navigation_handle_map_.end())
+    return;
+
+  NavigationEvent* nav_event = &navigation_handle_map_[navigation_handle];
+  nav_event->is_server_redirect = true;
+  GURL::Replacements replacements;
+  replacements.ClearRef();
+  nav_event->server_redirect_url =
+      navigation_handle->GetURL().ReplaceComponents(replacements);
+  nav_event->timestamp = base::Time::Now();
+}
+
+void SafeBrowsingNavigationObserver::DidFinishNavigation(
+    content::NavigationHandle* navigation_handle) {
+  if (navigation_handle_map_.find(navigation_handle) ==
+      navigation_handle_map_.end())
+    return;
+
+  // If it is an error page, we ignore this navigation.
+  if (navigation_handle->IsErrorPage()) {
+    navigation_handle_map_.erase(navigation_handle);
+    return;
+  }
+  GURL::Replacements replacements;
+  replacements.ClearRef();
+  NavigationEvent* nav_event = &navigation_handle_map_[navigation_handle];
+
+  // Set is_user_initiated to has_user_gesture || browser_initiated.
+  nav_event->is_user_initiated =
+      nav_event->is_user_initiated || !navigation_handle->IsRendererInitiated();
+  nav_event->has_committed = navigation_handle->HasCommitted();
+  nav_event->target_tab_id =
+      SessionTabHelper::IdForTab(navigation_handle->GetWebContents());
+  nav_event->timestamp = base::Time::Now();
+
+  GURL nav_event_key = navigation_handle->HasCommitted()
+                           ? navigation_handle->GetRenderFrameHost()
+                                 ->GetLastCommittedURL()
+                                 .ReplaceComponents(replacements)
+                           : nav_event->is_server_redirect
+                                 ? nav_event->server_redirect_url
+                                 : nav_event->target_url;
+  manager_->RecordNavigationEvent(nav_event_key, nav_event);
+  navigation_handle_map_.erase(navigation_handle);
+}
+
+void SafeBrowsingNavigationObserver::DidGetUserInteraction(
+    const blink::WebInputEvent::Type type) {
+  last_user_gesture_timestamp_ = base::Time::Now();
+  has_user_gesture_ = true;
+  // TODO (jialiul): Refine user gesture logic when DidOpenRequestedURL
+  // covers all retargetting cases.
+  manager_->RecordUserGestureForWebContents(web_contents(),
+                                            last_user_gesture_timestamp_);
+}
+
+void SafeBrowsingNavigationObserver::WebContentsDestroyed() {
+  manager_->OnWebContentDestroyed(web_contents());
+  web_contents()->RemoveUserData(kWebContentsUserDataKey);
+  // web_contents is null after this function.
+}
+
+}  // namespace safe_browsing