Add SafeBrowsingNavigationObserver to listen to navigation events

chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc

Index: chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc
diff --git a/chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc b/chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc
new file mode 100644
index 0000000000000000000000000000000000000000..3b6cd98fbadc6f129abdce588a5a2f384a719c1e
--- /dev/null
+++ b/chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc
@@ -0,0 +1,277 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+#include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h"
+
+#include "base/memory/ptr_util.h"
+#include "base/time/time.h"
+#include "chrome/browser/chrome_notification_types.h"
+#include "chrome/browser/sessions/session_tab_helper.h"
+#include "chrome/browser/tab_contents/retargeting_details.h"
+#include "content/public/browser/navigation_controller.h"
+#include "content/public/browser/navigation_details.h"
+#include "content/public/browser/navigation_entry.h"
+#include "content/public/browser/navigation_handle.h"
+#include "content/public/browser/notification_service.h"
+#include "content/public/browser/notification_types.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/render_process_host.h"
+#include "content/public/browser/resource_request_details.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/common/resource_type.h"
+#include "ui/base/page_transition_types.h"
+
+namespace safe_browsing {
+
+namespace {
+
+// Returns |time| as milliseconds since the epoch.
+double MilliSecondsFromTime(const base::Time& time) {
+  return 1000 * time.ToDoubleT();
+}
+
+}  // namespace
+
+// SafeBrowsingNavigationObserver::NavigationEvent-----------------------------
+
+SafeBrowsingNavigationObserver::NavigationEvent::NavigationEvent()
+    : source_url(),
+      source_tab_id(-1),
+      target_url(),
+      target_tab_id(-1),
+      frame_id(-1),
+      main_frame_url(),
+      timestamp(0.0),
+      is_user_initiated(false),
+      has_committed(false),
+      is_server_redirect(false),
+      is_finished(false),
+      server_redirect_url() {}
+
+SafeBrowsingNavigationObserver::NavigationEvent::NavigationEvent(
+    const GURL& source_url,
+    int source_tab_id,
+    const GURL& target_url,
+    int target_tab_id,
+    int frame_id,
+    const GURL& main_frame_url,
+    double timestamp,
+    bool is_user_initiated,
+    bool has_committed)
+    : source_url(source_url),
+      source_tab_id(source_tab_id),
+      target_url(target_url),
+      target_tab_id(target_tab_id),
+      frame_id(frame_id),
+      main_frame_url(main_frame_url),
+      timestamp(timestamp),
+      is_user_initiated(is_user_initiated),
+      has_committed(has_committed),
+      is_server_redirect(false),
+      is_finished(false),
+      server_redirect_url() {}
+
+SafeBrowsingNavigationObserver::NavigationEvent::NavigationEvent(

[Nathan Parker, 2016/09/03 00:21:35]

Does the default copy constructor work here?  I think there's a "= default"
syntax that invokes that.  I haven't used it.,

[Jialiu Lin, 2016/09/07 00:42:40]

Actually I need a move constructor here instead of a copy constructor. Fixed. 

+    const NavigationEvent& nav_event)
+    : source_url(nav_event.source_url),
+      source_tab_id(nav_event.source_tab_id),
+      target_url(nav_event.target_url),
+      target_tab_id(nav_event.target_tab_id),
+      frame_id(nav_event.frame_id),
+      main_frame_url(nav_event.main_frame_url),
+      timestamp(nav_event.timestamp),
+      is_user_initiated(nav_event.is_user_initiated),
+      has_committed(nav_event.has_committed),
+      is_server_redirect(nav_event.is_server_redirect),
+      is_finished(nav_event.is_finished),
+      server_redirect_url(nav_event.server_redirect_url) {}
+
+SafeBrowsingNavigationObserver::NavigationEvent::~NavigationEvent() {}
+
+SafeBrowsingNavigationObserver::SafeBrowsingNavigationObserver() {
+  registrar_.Add(this, content::NOTIFICATION_NAV_ENTRY_PENDING,
+                 content::NotificationService::AllSources());
+  registrar_.Add(this, chrome::NOTIFICATION_TAB_ADDED,
+                 content::NotificationService::AllSources());
+  registrar_.Add(this, chrome::NOTIFICATION_TAB_CLOSING,
+                 content::NotificationService::AllSources());
+  registrar_.Add(this, chrome::NOTIFICATION_RETARGETING,
+                 content::NotificationService::AllSources());
+}
+
+SafeBrowsingNavigationObserver::~SafeBrowsingNavigationObserver() {
+  ClearNavigationMap();
+}
+
+SafeBrowsingNavigationObserver::NavigationMap*
+SafeBrowsingNavigationObserver::navigation_map() {
+  return &navigation_map_;

[Nathan Parker, 2016/09/03 00:21:35]

nit: trivial accessor could go in the .h

[Jialiu Lin, 2016/09/07 00:42:40]

Done.

+}
+
+void SafeBrowsingNavigationObserver::ClearNavigationMap() {
+  navigation_map_.clear();
+}
+
+void SafeBrowsingNavigationObserver::RecordNavigationPendingEntry(
+    const content::NotificationSource& source,
+    const content::NotificationDetails& details) {
+  content::NavigationController* source_controller =
+      content::Source<content::NavigationController>(source).ptr();
+  DCHECK(source_controller);
+  content::WebContents* source_contents = source_controller->GetWebContents();
+  DCHECK(source_contents);
+  WebContentsObserver::Observe(source_contents);

[Nathan Parker, 2016/09/03 00:21:35]

What does this do?  Does it invoke it on *this?

[Jialiu Lin, 2016/09/07 00:42:41]

Yes, since SBNavigationObserver inherits both WebContentsObserver and
NotificationObserver, this is to explicitly say which Observe() function to
call. 

Only after a webcontents being observed, NavigationObserver can receive
DidStartNavigation/ DidFinishNavigaiton and other callbacks for this specific
webcontents.

+}
+
+void SafeBrowsingNavigationObserver::RecordRetargeting(
+    const content::NotificationDetails& details) {
+  const RetargetingDetails* retargeting_detail =
+      content::Details<const RetargetingDetails>(details).ptr();
+  DCHECK(retargeting_detail);
+  content::WebContents* source_contents =
+      retargeting_detail->source_web_contents;
+  content::WebContents* target_contents =
+      retargeting_detail->target_web_contents;
+  DCHECK(source_contents);
+  DCHECK(target_contents);
+  // WebContentsObserver::Observe(target_contents);
+
+  content::RenderFrameHost* rfh = content::RenderFrameHost::FromID(
+      source_contents->GetRenderProcessHost()->GetID(),
+      retargeting_detail->source_render_frame_id);
+  GURL target_url = retargeting_detail->target_url;

[Nathan Parker, 2016/09/03 00:21:35]

nit: could be const GURL&

[Jialiu Lin, 2016/09/07 00:42:41]

Indeed. done.

+
+  if (navigation_map_.find(target_url) == navigation_map_.end())
+    navigation_map_.insert(
+        std::make_pair(target_url, std::vector<NavigationEvent>()));
+
+  NavigationEvent nav_event(
+      rfh ? rfh->GetLastCommittedURL() : GURL(),    // source_url
+      SessionTabHelper::IdForTab(source_contents),  // source_tab_id
+      target_url,                                   // target_url
+      SessionTabHelper::IdForTab(target_contents),  // target_tab_id
+      rfh ? rfh->GetFrameTreeNodeId() : -1,         // frame id
+      source_contents->GetVisibleURL(),             // main_frame_url
+      MilliSecondsFromTime(base::Time::Now()),      // timestamp
+      true,                                         // is_user_initiated
+      false);                                       // is_committed
+
+  navigation_map_[target_url].push_back(std::move(nav_event));

[Nathan Parker, 2016/09/03 00:21:35]

I'm trying to remember the move semantics with std::vectors... Does it destroy
the nav_event that's on the stack?

[Jialiu Lin, 2016/09/07 00:42:41]

It basically pushes the underlying nav_event object into the vector without
making a copy. Yes, nav_event will be invalid after std::move.

+}
+
+void SafeBrowsingNavigationObserver::Observe(
+    int type,
+    const content::NotificationSource& source,
+    const content::NotificationDetails& details) {
+  switch (type) {
+    case content::NOTIFICATION_NAV_ENTRY_PENDING: {
+      RecordNavigationPendingEntry(source, details);
+      break;
+    }
+    case chrome::NOTIFICATION_RETARGETING: {
+      RecordRetargeting(details);
+      break;
+    }
+    case chrome::NOTIFICATION_TAB_ADDED: {
+      content::WebContents* dest_content =
+          content::Details<content::WebContents>(details).ptr();
+      DCHECK(dest_content);
+      WebContentsObserver::Observe(dest_content);
+      break;
+    }
+    case chrome::NOTIFICATION_TAB_CLOSING: {
+      // TODO(jialiul): For now, we don't want to delete NavigationEvent if
+      // its corresponding tab is closed, since it could be used to hide landing
+      // page. But we may consider using this event as a trigger to clean up
+      // some outdated entries.

[Nathan Parker, 2016/09/03 00:21:35]

Is it useful to record this event?

[Jialiu Lin, 2016/09/07 00:42:41]

Maybe. Let me handle or delete this in my next CL.

+      break;
+    }
+    default:
+      NOTREACHED();
+      break;
+  }
+}
+
+void SafeBrowsingNavigationObserver::DidStartNavigation(

[Nathan Parker, 2016/09/03 00:21:35]

For those of us not familiar with NavigationObserver, it would be useful to add
a brief comment on when/where/why this gets called.

[Jialiu Lin, 2016/09/07 00:42:40]

Comments added.

+    content::NavigationHandle* navigation_handle) {
+  // If we already seen this navigation_handle before, no need to do anything.
+  if (navigation_handle_map_.find(navigation_handle) !=
+      navigation_handle_map_.end())
+    return;
+
+  // Construct a NavigationEvent based on avialble information in

[Nathan Parker, 2016/09/03 00:21:35]

sp: available

[Jialiu Lin, 2016/09/07 00:42:40]

done

+  // navigation_handle.
+  NavigationEvent nav_event;
+  content::RenderFrameHost* host =
+      navigation_handle->GetWebContents()->FindFrameByFrameTreeNodeId(
+          navigation_handle->GetFrameTreeNodeId());
+  if (host && host->GetLastCommittedURL() != GURL())

[Nathan Parker, 2016/09/03 00:21:35]

Would GetLasCommittedURL().is_valid() work?  Avoids creating a temporary.

[Jialiu Lin, 2016/09/07 00:42:40]

Done. 

+    nav_event.source_url = host->GetLastCommittedURL();
+  else

[Nathan Parker, 2016/09/03 00:21:34]

What's the difference between the source_url in these two cases?

[Jialiu Lin, 2016/09/07 00:42:41]

The differences are whether this is the first URL to be commit in this frame or
not. Comments added to make it clearer. 

+    nav_event.source_url = navigation_handle->GetURL();
+  nav_event.target_url = navigation_handle->GetURL();
+
+  nav_event.source_tab_id =
+      SessionTabHelper::IdForTab(navigation_handle->GetWebContents());
+  nav_event.timestamp = MilliSecondsFromTime(base::Time::Now());
+  nav_event.frame_id = navigation_handle->GetFrameTreeNodeId();
+
+  if (navigation_handle->IsInMainFrame()) {

[Nathan Parker, 2016/09/03 00:21:35]

Do you want to record in the nav_event whether or not it's in main frame?

[Jialiu Lin, 2016/09/07 00:42:40]

frame_id  and main_frame_url are recorded, which are enough to tell whether it
is a mainframe navigation or not. 

+    nav_event.main_frame_url = nav_event.source_url;
+  } else {
+    nav_event.main_frame_url = navigation_handle->GetWebContents()->GetURL();

[Nathan Parker, 2016/09/03 00:21:35]

nit: this uses braces for single-line if/else, but above does not.  Just be
consistent.

[Jialiu Lin, 2016/09/07 00:42:40]

Thanks for catching this.

+  }
+  navigation_handle_map_[navigation_handle] = std::move(nav_event);
+}
+
+void SafeBrowsingNavigationObserver::DidRedirectNavigation(
+    content::NavigationHandle* navigation_handle) {
+  // We should have already seen this navigation_handle in DidStartNavigation.
+  if (navigation_handle_map_.find(navigation_handle) ==
+      navigation_handle_map_.end())
+    return;
+  NavigationEvent* nav_event = &navigation_handle_map_[navigation_handle];
+  nav_event->is_server_redirect = true;
+  nav_event->server_redirect_url = navigation_handle->GetURL();
+  nav_event->timestamp = MilliSecondsFromTime(base::Time::Now());
+}
+
+void SafeBrowsingNavigationObserver::DidFinishNavigation(
+    content::NavigationHandle* navigation_handle) {
+  if (navigation_handle_map_.find(navigation_handle) ==
+      navigation_handle_map_.end())
+    return;
+  NavigationEvent* nav_event = &navigation_handle_map_[navigation_handle];
+
+  // In NavigationHandle's definition, render initiated navigation includes:
+  // clicking <a> link, changing window.location.href, client redirect via
+  // meta refresh tag and using window.history.pushState.
+  // Though in our case, we consider user explicitly clicking on a link as user
+  // initiated. We first use the negate of IsRenderInitiated() to include
+  // user initiated navigations such as typing in the omni box or clicking on
+  // bookmark then we refine this estimate by using HasUserGesture()
+  // if this info is available.
+  // For non-committed navigations (e.g. downloads),
+  // NavigationHandle::HasUserGesture() is not availble. But we can later get it
+  // from DownloadItem::HasUserGesture().
+  nav_event->is_user_initiated = !navigation_handle->IsRendererInitiated();
+  if (navigation_handle->HasCommitted()) {
+    nav_event->has_committed = true;
+    nav_event->is_user_initiated =
+        nav_event->is_user_initiated || navigation_handle->HasUserGesture();
+  }
+  nav_event->target_tab_id =
+      SessionTabHelper::IdForTab(navigation_handle->GetWebContents());
+  nav_event->timestamp = MilliSecondsFromTime(base::Time::Now());
+  nav_event->is_finished = true;
+
+  GURL key_url = nav_event->is_server_redirect ? nav_event->server_redirect_url
+                                               : nav_event->target_url;
+  if (navigation_map_.find(key_url) == navigation_map_.end())
+    navigation_map_[key_url] = std::vector<NavigationEvent>();

[Nathan Parker, 2016/09/03 00:21:35]

nit: Can you insert a pair of key_url and
std::vector<NavigationEvent>({*nav_event})?
I don't think I have the syntax quite right, but it'd be a bit more efficient. 
If it's ugly, never mind.

[Jialiu Lin, 2016/09/07 00:42:41]

Yep, insert is safer than [] operator. 

+  navigation_map_[key_url].push_back(std::move(*nav_event));
+
+  navigation_handle_map_.erase(navigation_handle);
+}
+
+}  // namespace safe_browsing