Add SafeBrowsingNavigationObserver to listen to navigation events

chrome/browser/safe_browsing/safe_browsing_navigation_observer.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h"
+
+#include "base/memory/ptr_util.h"
+#include "base/time/time.h"
+#include "chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager.h"
+#include "chrome/browser/sessions/session_tab_helper.h"
+#include "content/public/browser/navigation_handle.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/resource_request_details.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/common/resource_type.h"
+
+using content::WebContents;
+
+namespace {
+const char kWebContentsUserDataKey[] =
+    "web_contents_safe_browsing_navigation_observer";
+}  // namespace
+
+namespace safe_browsing {
+
+// SafeBrowsingNavigationObserver::NavigationEvent-----------------------------
+NavigationEvent::NavigationEvent()
+    : source_url(),
+      source_tab_id(-1),
+      source_main_frame_url(),
+      original_request_url(),
+      target_tab_id(-1),
+      frame_id(-1),
+      last_updated(base::Time::Now()),
+      is_user_initiated(false),
+      has_committed(false),
+      has_server_redirect(false),
+      destination_url() {}
+
+NavigationEvent::NavigationEvent(NavigationEvent&& nav_event)
+    : source_url(std::move(nav_event.source_url)),
+      source_tab_id(std::move(nav_event.source_tab_id)),
+      source_main_frame_url(std::move(nav_event.source_main_frame_url)),
+      original_request_url(std::move(nav_event.original_request_url)),
+      target_tab_id(std::move(nav_event.target_tab_id)),
+      frame_id(nav_event.frame_id),
+      last_updated(nav_event.last_updated),
+      is_user_initiated(nav_event.is_user_initiated),
+      has_committed(nav_event.has_committed),
+      has_server_redirect(nav_event.has_server_redirect),
+      destination_url(std::move(nav_event.destination_url)) {}
+
+NavigationEvent& NavigationEvent::operator=(NavigationEvent&& nav_event) {
+  source_url = std::move(nav_event.source_url);
+  source_tab_id = nav_event.source_tab_id;
+  source_main_frame_url = std::move(nav_event.source_main_frame_url);
+  original_request_url = std::move(nav_event.original_request_url);
+  target_tab_id = nav_event.target_tab_id;
+  frame_id = nav_event.frame_id;
+  last_updated = nav_event.last_updated;
+  is_user_initiated = nav_event.is_user_initiated;
+  has_committed = nav_event.has_committed;
+  has_server_redirect = nav_event.has_server_redirect;
+  destination_url = std::move(nav_event.destination_url);
+  return *this;
+}
+
+NavigationEvent::~NavigationEvent() {}
+
+// SafeBrowsingNavigationObserver --------------------------------------------
+
+// static
+void SafeBrowsingNavigationObserver::MaybeCreateForWebContents(
+    content::WebContents* web_contents) {
+  if (FromWebContents(web_contents))
+    return;
+  NOTIMPLEMENTED();
+  // TODO(jialiul): This method will be called by TabHelpers::AttachTabHelpers.

[Charlie Reis, 2016/10/25 04:52:26]

nit: TODO above NOTIMPLEMENTED

[Jialiu Lin, 2016/10/27 02:15:19]

Done.

+  // Complete this method when the entire class is ready.
+}
+
+// static
+SafeBrowsingNavigationObserver* SafeBrowsingNavigationObserver::FromWebContents(
+    content::WebContents* web_contents) {
+  return static_cast<SafeBrowsingNavigationObserver*>(
+      web_contents->GetUserData(kWebContentsUserDataKey));
+}
+
+SafeBrowsingNavigationObserver::SafeBrowsingNavigationObserver(
+    content::WebContents* contents,
+    const scoped_refptr<SafeBrowsingNavigationObserverManager>& manager)
+    : content::WebContentsObserver(contents),
+      manager_(manager),
+      has_user_gesture_(false),
+      last_user_gesture_timestamp_(base::Time()) {}
+
+SafeBrowsingNavigationObserver::~SafeBrowsingNavigationObserver() {}
+
+// Called when a navigation started in the WebContents. |navigation_handle| in

[Charlie Reis, 2016/10/25 04:52:26]

nit: s/started/starts/
nit: s/in parameter/parameter/
  (there's no ambiguity, since there aren't any out parameters)

[Jialiu Lin, 2016/10/27 02:15:19]

Done.

+// parameter is unique to this navigation, which will appear in the following
+// DidRedirectNavigation, and DidFinishNavigation too.
+void SafeBrowsingNavigationObserver::DidStartNavigation(
+    content::NavigationHandle* navigation_handle) {
+  NavigationEvent nav_event;
+  auto it = navigation_handle_map_.find(navigation_handle);
+  // It is possible to see multiple DidStartNaivgation(..) with the same

[Charlie Reis, 2016/10/25 04:52:26]

nit: multiple DidStartNavigation calls
(Note the "Naiv" typo)

[Jialiu Lin, 2016/10/27 02:15:19]

Good eye!

+  // navigation_handle (e.g. cross-process transfer). If that's the case,
+  // we need to copy the is_user_initiated field.
+  if (it != navigation_handle_map_.end()) {
+    nav_event.is_user_initiated = it->second.is_user_initiated;
+  } else {
+    // If this is the first time we see this navigation_handle, create a new
+    // NavigationEvent, and decide if it is triggered by user.
+    if ((has_user_gesture_ &&
+         !SafeBrowsingNavigationObserverManager::IsUserGestureExpired(
+             last_user_gesture_timestamp_)) ||
+        !navigation_handle->IsRendererInitiated()) {
+      nav_event.is_user_initiated = has_user_gesture_;
+      manager_->OnUserGestureConsumed(web_contents(),
+                                      last_user_gesture_timestamp_);
+    }
+    has_user_gesture_ = false;
+  }
+
+  // All the other fields are reconstructed based on current content of
+  // navigation_handle.
+  nav_event.frame_id = navigation_handle->GetFrameTreeNodeId();
+  content::RenderFrameHost* current_frame_host =

[Charlie Reis, 2016/10/25 04:52:26]

nit: Move this declaration under the comment below, since it should go away when
we resolve the TODO.

[Jialiu Lin, 2016/10/27 02:15:19]

Done.

+      navigation_handle->GetWebContents()->FindFrameByFrameTreeNodeId(
+          nav_event.frame_id);
+
+  // If there was URL previously committed in the current RenderFrameHost,

[Charlie Reis, 2016/10/25 04:52:26]

nit: a URL

[Jialiu Lin, 2016/10/27 02:15:19]

Done.

+  // set it as the source url of this navigation. Otherwise, this is the
+  // first url going to commit in this frame. We set navigation_handle's URL as
+  // the source url.
+  // TODO(jialiul): source_url will be inccorect when another frame is targeting

[Charlie Reis, 2016/10/25 04:52:26]

nit: incorrect

Maybe also mention source_tab_id and source_main_frame_url in the TODO?  I think
those have the same limitation, right?

[Jialiu Lin, 2016/10/27 02:15:20]

Yes, indeed. Comment refined.

+  // this frame. Need to refine this logic after the true initiator details are
+  // added to NavigationHandle (https://crbug.com/651895).
+  if (current_frame_host &&
+      current_frame_host->GetLastCommittedURL().is_valid()) {
+    nav_event.source_url = SafeBrowsingNavigationObserverManager::ClearEmptyRef(
+        current_frame_host->GetLastCommittedURL());
+  } else {
+    nav_event.source_url = SafeBrowsingNavigationObserverManager::ClearEmptyRef(
+        navigation_handle->GetURL());

[Charlie Reis, 2016/10/25 04:52:26]

I'm curious, is there anything that depends on the source URL being non-empty? 
It seems like a browser-initiated navigation might be best treated like it has
no source URL, since it may be unrelated to what the user is currently looking
at.  (Similarly, until the initiator URL plumbing is added, it's a bit unusual
to say that a URL is its own src URL for the first navigation in a frame.)

The only reason I mention is that it might be more difficult to analyze the data
if you have to always check whether the source_url is different from the
original_request_url before deciding whether some other page was responsible for
the navigation.  If you leave it empty in the cases that no other page was
responsible, it might be easier to analyze the data.

[Jialiu Lin, 2016/10/27 02:15:19]

Make sense. 

+  }
+  nav_event.original_request_url =
+      SafeBrowsingNavigationObserverManager::ClearEmptyRef(
+          navigation_handle->GetURL());
+  nav_event.destination_url = nav_event.original_request_url;
+
+  nav_event.source_tab_id =
+      SessionTabHelper::IdForTab(navigation_handle->GetWebContents());
+
+  if (navigation_handle->IsInMainFrame()) {
+    nav_event.source_main_frame_url = nav_event.source_url;
+  } else {
+    nav_event.source_main_frame_url =
+        SafeBrowsingNavigationObserverManager::ClearEmptyRef(
+            navigation_handle->GetWebContents()->GetLastCommittedURL());
+  }
+  navigation_handle_map_[navigation_handle] = std::move(nav_event);
+}
+
+void SafeBrowsingNavigationObserver::DidRedirectNavigation(
+    content::NavigationHandle* navigation_handle) {
+  // We should have already seen this navigation_handle in DidStartNavigation.
+  DCHECK(navigation_handle_map_.find(navigation_handle) !=
+         navigation_handle_map_.end());

[Charlie Reis, 2016/10/25 04:52:26]

You might want to rephrase this to a if (...) { NOTREACHED; return; }

That way we don't crash on line 172 if the handle isn't found in a release
build.  Same in DidFinishNavigation.

[Jialiu Lin, 2016/10/27 02:15:19]

Done.

+
+  NavigationEvent* nav_event = &navigation_handle_map_[navigation_handle];
+  nav_event->has_server_redirect = true;
+  nav_event->destination_url =
+      SafeBrowsingNavigationObserverManager::ClearEmptyRef(
+          navigation_handle->GetURL());
+  nav_event->last_updated = base::Time::Now();
+}
+
+void SafeBrowsingNavigationObserver::DidFinishNavigation(
+    content::NavigationHandle* navigation_handle) {
+  DCHECK(navigation_handle_map_.find(navigation_handle) !=
+         navigation_handle_map_.end());
+
+  // If it is an error page, we ignore this navigation.
+  if (navigation_handle->IsErrorPage()) {
+    navigation_handle_map_.erase(navigation_handle);
+    return;
+  }
+  NavigationEvent* nav_event = &navigation_handle_map_[navigation_handle];
+
+  nav_event->has_committed = navigation_handle->HasCommitted();
+  nav_event->target_tab_id =
+      SessionTabHelper::IdForTab(navigation_handle->GetWebContents());
+  nav_event->last_updated = base::Time::Now();
+
+  manager_->RecordNavigationEvent(nav_event->destination_url, nav_event);
+  navigation_handle_map_.erase(navigation_handle);
+}
+
+void SafeBrowsingNavigationObserver::DidGetResourceResponseStart(
+    const content::ResourceRequestDetails& details) {
+  // We only care about main frame and sub frame.
+  if (details.resource_type != content::RESOURCE_TYPE_MAIN_FRAME &&
+      details.resource_type != content::RESOURCE_TYPE_SUB_FRAME) {
+    return;
+  }
+  if (!details.url.is_valid() || details.socket_address.IsEmpty())
+    return;
+
+  manager_->RecordHostToIpMapping(details.url.host(),
+                                  details.socket_address.host());
+}
+
+void SafeBrowsingNavigationObserver::DidGetUserInteraction(
+    const blink::WebInputEvent::Type type) {
+  last_user_gesture_timestamp_ = base::Time::Now();
+  has_user_gesture_ = true;
+  // TODO (jialiul): Refine user gesture logic when DidOpenRequestedURL
+  // covers all retargetting cases.
+  manager_->RecordUserGestureForWebContents(web_contents(),
+                                            last_user_gesture_timestamp_);
+}
+
+void SafeBrowsingNavigationObserver::WebContentsDestroyed() {
+  manager_->OnWebContentDestroyed(web_contents());
+  web_contents()->RemoveUserData(kWebContentsUserDataKey);
+  // web_contents is null after this function.
+}
+
+}  // namespace safe_browsing