Add SafeBrowsingNavigationObserver to listen to navigation events

chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc

Index: chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc
diff --git a/chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc b/chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc
new file mode 100644
index 0000000000000000000000000000000000000000..d4514318befa424239faa6e280a3d52ec936bcea
--- /dev/null
+++ b/chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc
@@ -0,0 +1,276 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+#include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h"

[Charlie Reis, 2016/09/16 23:35:04]

nit: Add blank line before.

[Jialiu Lin, 2016/09/22 21:30:28]

Done.

+
+#include "base/memory/ptr_util.h"
+#include "base/time/time.h"
+#include "chrome/browser/chrome_notification_types.h"
+#include "chrome/browser/sessions/session_tab_helper.h"
+#include "chrome/browser/tab_contents/retargeting_details.h"
+#include "content/public/browser/navigation_controller.h"
+#include "content/public/browser/navigation_details.h"
+#include "content/public/browser/navigation_entry.h"
+#include "content/public/browser/navigation_handle.h"
+#include "content/public/browser/notification_service.h"
+#include "content/public/browser/notification_types.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/render_process_host.h"
+#include "content/public/browser/resource_request_details.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/common/resource_type.h"
+#include "ui/base/page_transition_types.h"
+
+namespace safe_browsing {
+
+namespace {
+
+// Returns |time| as seconds since the epoch.
+double SecondsFromTime(const base::Time& time) {

[Charlie Reis, 2016/09/16 23:35:04]

Why do we need this?

[Jialiu Lin, 2016/09/22 21:30:28]

removed.

+  return 1000000 * time.ToDoubleT();
+}
+
+}  // namespace
+
+// SafeBrowsingNavigationObserver::NavigationEvent-----------------------------
+
+SafeBrowsingNavigationObserver::NavigationEvent::NavigationEvent()
+    : source_url(),
+      source_tab_id(-1),
+      target_url(),
+      target_tab_id(-1),
+      frame_id(-1),
+      main_frame_url(),
+      timestamp(0.0),
+      is_user_initiated(false),
+      has_committed(false),
+      is_server_redirect(false),
+      is_finished(false),
+      server_redirect_url() {}
+
+SafeBrowsingNavigationObserver::NavigationEvent::NavigationEvent(
+    NavigationEvent&& nav_event)
+    : source_url(std::move(nav_event.source_url)),
+      source_tab_id(std::move(nav_event.source_tab_id)),
+      target_url(std::move(nav_event.target_url)),
+      target_tab_id(std::move(nav_event.target_tab_id)),
+      frame_id(nav_event.frame_id),
+      main_frame_url(std::move(nav_event.main_frame_url)),
+      timestamp(nav_event.timestamp),
+      is_user_initiated(nav_event.is_user_initiated),
+      has_committed(nav_event.has_committed),
+      is_server_redirect(nav_event.is_server_redirect),
+      is_finished(nav_event.is_finished),
+      server_redirect_url(std::move(nav_event.server_redirect_url)) {}
+
+SafeBrowsingNavigationObserver::NavigationEvent&
+SafeBrowsingNavigationObserver::NavigationEvent::operator=(
+    NavigationEvent&& nav_event) {
+  source_url = std::move(nav_event.source_url);
+  source_tab_id = std::move(nav_event.source_tab_id);
+  target_url = std::move(nav_event.target_url);
+  target_tab_id = std::move(nav_event.target_tab_id);
+  frame_id = nav_event.frame_id;
+  main_frame_url = std::move(nav_event.main_frame_url);
+  timestamp = nav_event.timestamp;
+  is_user_initiated = nav_event.is_user_initiated;
+  has_committed = nav_event.has_committed;
+  is_server_redirect = nav_event.is_server_redirect;
+  is_finished = nav_event.is_finished;
+  server_redirect_url = std::move(nav_event.server_redirect_url);
+  return *this;
+}
+
+SafeBrowsingNavigationObserver::NavigationEvent::~NavigationEvent() {}
+
+SafeBrowsingNavigationObserver::SafeBrowsingNavigationObserver() {
+  registrar_.Add(this, content::NOTIFICATION_NAV_ENTRY_PENDING,

[Charlie Reis, 2016/09/16 23:35:05]

Why do you need this one?  The fewer notifications we can rely on, the better.

[Jialiu Lin, 2016/09/22 21:30:28]

Removed, only keep NOTIFICATION_RETARGETING

+                 content::NotificationService::AllSources());
+  registrar_.Add(this, chrome::NOTIFICATION_TAB_ADDED,
+                 content::NotificationService::AllSources());
+  registrar_.Add(this, chrome::NOTIFICATION_TAB_CLOSING,
+                 content::NotificationService::AllSources());
+  registrar_.Add(this, chrome::NOTIFICATION_RETARGETING,
+                 content::NotificationService::AllSources());
+}
+
+SafeBrowsingNavigationObserver::~SafeBrowsingNavigationObserver() {
+  ClearNavigationMap();
+}
+
+void SafeBrowsingNavigationObserver::ClearNavigationMap() {
+  navigation_map_.clear();
+}
+
+void SafeBrowsingNavigationObserver::RecordNavigationPendingEntry(
+    const content::NotificationSource& source,
+    const content::NotificationDetails& details) {
+  content::NavigationController* source_controller =
+      content::Source<content::NavigationController>(source).ptr();
+  DCHECK(source_controller);
+  content::WebContents* source_contents = source_controller->GetWebContents();
+  DCHECK(source_contents);
+  WebContentsObserver::Observe(source_contents);
+}
+
+void SafeBrowsingNavigationObserver::RecordRetargeting(
+    const content::NotificationDetails& details) {
+  const RetargetingDetails* retargeting_detail =
+      content::Details<const RetargetingDetails>(details).ptr();
+  DCHECK(retargeting_detail);
+  content::WebContents* source_contents =
+      retargeting_detail->source_web_contents;
+  content::WebContents* target_contents =
+      retargeting_detail->target_web_contents;
+  DCHECK(source_contents);
+  DCHECK(target_contents);
+
+  content::RenderFrameHost* rfh = content::RenderFrameHost::FromID(
+      source_contents->GetRenderProcessHost()->GetID(),
+      retargeting_detail->source_render_frame_id);
+  const GURL& target_url = retargeting_detail->target_url;
+
+  navigation_map_.insert(
+      std::make_pair(target_url, std::vector<NavigationEvent>()));
+
+  NavigationEvent nav_event;
+  nav_event.source_url = rfh ? rfh->GetLastCommittedURL() : GURL();
+  nav_event.source_tab_id = SessionTabHelper::IdForTab(source_contents);
+  nav_event.target_url = target_url;
+  nav_event.target_tab_id = SessionTabHelper::IdForTab(target_contents);
+  nav_event.frame_id = rfh ? rfh->GetFrameTreeNodeId() : -1;
+  nav_event.main_frame_url = source_contents->GetVisibleURL();
+  nav_event.timestamp = SecondsFromTime(base::Time::Now());
+  nav_event.is_user_initiated = true;
+
+  navigation_map_[target_url].push_back(std::move(nav_event));
+}
+
+void SafeBrowsingNavigationObserver::Observe(
+    int type,
+    const content::NotificationSource& source,
+    const content::NotificationDetails& details) {
+  switch (type) {
+    case content::NOTIFICATION_NAV_ENTRY_PENDING: {
+      RecordNavigationPendingEntry(source, details);
+      break;
+    }
+    case chrome::NOTIFICATION_RETARGETING: {
+      RecordRetargeting(details);
+      break;
+    }
+    case chrome::NOTIFICATION_TAB_ADDED: {
+      content::WebContents* dest_content =
+          content::Details<content::WebContents>(details).ptr();
+      DCHECK(dest_content);
+      WebContentsObserver::Observe(dest_content);

[Charlie Reis, 2016/09/16 23:35:05]

This will stop observing the previous WebContents, so it won't work.  Nick
points out that TabHelpers::AttachTabHelpers is the right pattern to use for
listening to all WebContents (where we create one observer per WebContents).

[Jialiu Lin, 2016/09/22 21:30:28]

Done.

+      break;
+    }
+    case chrome::NOTIFICATION_TAB_CLOSING: {
+      // TODO(jialiul): For now, we don't want to delete NavigationEvent if
+      // its corresponding tab is closed, since it could be used to hide landing
+      // page. But we may consider using this event as a trigger to clean up
+      // some outdated entries.
+      break;
+    }
+    default:
+      NOTREACHED();
+      break;
+  }
+}
+
+// -----------------content::WebContentsObserver overrides--------------
+// Called when a navigation started in the WebContents. |navigation_handle| in
+// paramter is unique to this navigation, which will appear in the following
+// DidRedirectNavigation, and DidFinishNavigation call too.
+void SafeBrowsingNavigationObserver::DidStartNavigation(
+    content::NavigationHandle* navigation_handle) {
+  // If we already seen this navigation_handle before, no need to do anything.
+  if (navigation_handle_map_.find(navigation_handle) !=
+      navigation_handle_map_.end())
+    return;
+
+  // Construct a NavigationEvent based on available information in
+  // navigation_handle.
+  NavigationEvent nav_event;
+  content::RenderFrameHost* host =
+      navigation_handle->GetWebContents()->FindFrameByFrameTreeNodeId(
+          navigation_handle->GetFrameTreeNodeId());
+
+  // If there was URL previously committed in this render frame host,
+  // set it as the source url of this navigation. Otherwise, this is the
+  // first url going to commit in this frame. We set navigation_handle's URL as
+  // the source url.

[Charlie Reis, 2016/09/16 23:35:05]

This doesn't sound right to me.  If a navigation in one frame/tab targets
another frame/tab, the DidStartNavigation will pass a handle for the new
frame/tab.  That means you wouldn't get the URL of the frame that initiated the
navigation.

I thought this was the reason we wanted to use Jochen's extension API
suggestion?  (I'm having trouble remembering the details.)

[Jialiu Lin, 2016/09/22 21:30:28]

retargeting is handled by listening to chrome::NOTIFICATION_RETARGETING
notification in SBNavigationObserverManager.
This is similar to what jochen's extension API did, though unfortunately I
cannot use his classes since they are defined in chrome/browser/extensions/api

+  if (host && host->GetLastCommittedURL().is_valid())
+    nav_event.source_url = host->GetLastCommittedURL();
+  else
+    nav_event.source_url = navigation_handle->GetURL();
+
+  nav_event.target_url = navigation_handle->GetURL();
+
+  nav_event.source_tab_id =
+      SessionTabHelper::IdForTab(navigation_handle->GetWebContents());
+  nav_event.timestamp = SecondsFromTime(base::Time::Now());
+  nav_event.frame_id = navigation_handle->GetFrameTreeNodeId();
+
+  if (navigation_handle->IsInMainFrame())
+    nav_event.main_frame_url = nav_event.source_url;
+  else
+    nav_event.main_frame_url = navigation_handle->GetWebContents()->GetURL();

[Charlie Reis, 2016/09/16 23:35:04]

GetURL is deprecated because it's ambiguous whether you want the visible or last
committed one.  I think you want GetLastCommittedURL.

[Jialiu Lin, 2016/09/22 21:30:28]

Done.

+
+  navigation_handle_map_.insert(
+      std::make_pair(navigation_handle, std::move(nav_event)));
+}
+
+void SafeBrowsingNavigationObserver::DidRedirectNavigation(
+    content::NavigationHandle* navigation_handle) {
+  // We should have already seen this navigation_handle in DidStartNavigation.
+  if (navigation_handle_map_.find(navigation_handle) ==
+      navigation_handle_map_.end())
+    return;
+  NavigationEvent* nav_event = &navigation_handle_map_[navigation_handle];
+  nav_event->is_server_redirect = true;
+  nav_event->server_redirect_url = navigation_handle->GetURL();

[Charlie Reis, 2016/09/16 23:35:04]

If you only want to track the last redirect and not the chain, we should
document that in the declaration.

[Jialiu Lin, 2016/09/22 21:30:28]

Done.

+  nav_event->timestamp = SecondsFromTime(base::Time::Now());
+}
+
+void SafeBrowsingNavigationObserver::DidFinishNavigation(
+    content::NavigationHandle* navigation_handle) {
+  if (navigation_handle_map_.find(navigation_handle) ==
+      navigation_handle_map_.end())
+    return;
+  NavigationEvent* nav_event = &navigation_handle_map_[navigation_handle];
+
+  // In NavigationHandle's definition, render initiated navigation includes:

[Charlie Reis, 2016/09/16 23:35:05]

nit: s/render/renderer/

[Jialiu Lin, 2016/09/22 21:30:28]

Done.

+  // clicking <a> link, changing window.location.href, client redirect via
+  // meta refresh tag and using window.history.pushState.

[Charlie Reis, 2016/09/16 23:35:04]

Be careful not to imply that this list exhaustive.  It's a very small list of
examples that try to give a sense for the variety of ways the renderer process
can navigate.

[Jialiu Lin, 2016/09/22 21:30:28]

Fixed comments

+  // Though in our case, we consider user explicitly clicking on a link as user
+  // initiated. We first use the negate of IsRenderInitiated() to include
+  // user initiated navigations such as typing in the omni box or clicking on
+  // bookmark then we refine this estimate by using HasUserGesture()
+  // if this info is available.

[Charlie Reis, 2016/09/16 23:35:04]

Yes, we generally consider "user initiated" to mean browser_initiated ||
has_user_gesture.  You can probably shorten the description a bit accordingly.

[Jialiu Lin, 2016/09/22 21:30:28]

Done.

+  // For non-committed navigations (e.g. downloads),
+  // NavigationHandle::HasUserGesture() is not availble. But we can later get it
+  // from DownloadItem::HasUserGesture().
+  nav_event->is_user_initiated = !navigation_handle->IsRendererInitiated();
+  if (navigation_handle->HasCommitted()) {
+    nav_event->has_committed = true;
+    nav_event->is_user_initiated =
+        nav_event->is_user_initiated || navigation_handle->HasUserGesture();
+  }
+  nav_event->target_tab_id =
+      SessionTabHelper::IdForTab(navigation_handle->GetWebContents());
+  nav_event->timestamp = SecondsFromTime(base::Time::Now());
+  nav_event->is_finished = true;
+
+  GURL key_url = nav_event->is_server_redirect ? nav_event->server_redirect_url
+                                               : nav_event->target_url;

[Charlie Reis, 2016/09/16 23:35:04]

I think we might need to be more careful about the URL than this.  It's not
clear to me whether navigation_handle->GetURL() will match either of these at
the time it finishes (e.g., if it ends up being an error page, blocked by CSP,
etc).  Maybe that gets handled as a new DidStart/DidFinish?  (Nasko is looking
into this at the moment, I believe.)

It would also help to have more documentation on what you expect target_url and
server_redirect_url to be.  Is target_url supposed to match the committed URL if
there's a redirect, or is it really more of an original URL for the request?

[Jialiu Lin, 2016/09/22 21:30:28]

You're right. I should use the committed URL instead. In case of an error page,
I can
probably ignore this navigation, since it is unlikely to trigger a download.

+  navigation_map_.insert(
+      std::make_pair(key_url, std::vector<NavigationEvent>()));
+
+  navigation_map_.at(key_url).push_back(std::move(*nav_event));

[Charlie Reis, 2016/09/16 23:35:05]

Why do this separately (requiring a second lookup)?

[Jialiu Lin, 2016/09/22 21:30:28]

Done.

+
+  navigation_handle_map_.erase(navigation_handle);
+}
+
+}  // namespace safe_browsing