Add SafeBrowsingNavigationObserver to listen to navigation events

chrome/browser/safe_browsing/safe_browsing_navigation_observer_browsertest.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/strings/stringprintf.h"
+#include "chrome/browser/chrome_notification_types.h"
+#include "chrome/browser/download/download_prefs.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/browser/safe_browsing/safe_browsing_navigation_observer.h"
+#include "chrome/browser/safe_browsing/safe_browsing_navigation_observer_manager.h"
+#include "chrome/browser/sessions/session_tab_helper.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/browser/ui/tabs/tab_strip_model.h"
+#include "chrome/common/pref_names.h"
+#include "chrome/test/base/in_process_browser_test.h"
+#include "chrome/test/base/ui_test_utils.h"
+#include "components/prefs/pref_service.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/download_item.h"
+#include "content/public/browser/download_manager.h"
+#include "content/public/browser/notification_service.h"
+#include "content/public/test/browser_test_utils.h"
+#include "content/public/test/test_utils.h"
+#include "net/dns/mock_host_resolver.h"
+#include "net/test/embedded_test_server/embedded_test_server.h"
+#include "url/gurl.h"
+#include "url/url_canon.h"
+
+namespace safe_browsing {
+
+const char kSingleFrameTestURL[] =
+    "/safe_browsing/download_protection/navigation_observer/"
+    "navigation_observer_tests.html";
+const char kMultiFrameTestURL[] =
+    "/safe_browsing/download_protection/navigation_observer/"
+    "navigation_observer_multi_frame_tests.html";
+const char kRedirectURL[] =
+    "/safe_browsing/download_protection/navigation_observer/redirect.html";
+const char kDownloadDataURL[] =
+    "data:application/octet-stream;base64,a2poYWxrc2hkbGtoYXNka2xoYXNsa2RoYWxra"
+    "GtoYWxza2hka2xzamFoZGxramhhc2xka2hhc2xrZGgKYXNrZGpoa2FzZGpoYWtzaGRrYXNoZGt"
+    "oYXNrZGhhc2tkaGthc2hka2Foc2RraGFrc2hka2FzaGRraGFzCmFza2pkaGFrc2hkbSxjbmtza"
+    "mFoZGtoYXNrZGhhc2tka2hrYXNkCjg3MzQ2ODEyNzQ2OGtqc2hka2FoZHNrZGhraApha3NqZGt"
+    "hc2Roa3NkaGthc2hka2FzaGtkaAohISomXkAqJl4qYWhpZGFzeWRpeWlhc1xcb1wKa2Fqc2Roa"
+    "2FzaGRrYXNoZGsKYWtzamRoc2tkaAplbmQK";
+const char kIframeDirectDownloadURL[] =
+    "/safe_browsing/download_protection/navigation_observer/iframe.html";
+const char kIframeRetargetingURL[] =
+    "/safe_browsing/download_protection/navigation_observer/"
+    "iframe_retargeting.html";
+const char kDownloadItemURL[] = "/safe_browsing/download_protection/signed.exe";
+
+// Test class to help create SafeBrowsingNavigationObservers for each
+// WebContents before they are actually installed through AttachTabHelper.
+class TestNavigationObserverManager
+    : public SafeBrowsingNavigationObserverManager {
+ public:
+  TestNavigationObserverManager() : SafeBrowsingNavigationObserverManager() {
+    registrar_.Add(this, chrome::NOTIFICATION_TAB_ADDED,
+                   content::NotificationService::AllSources());
+  }
+
+  void Observe(int type,
+               const content::NotificationSource& source,
+               const content::NotificationDetails& details) override {
+    if (type == chrome::NOTIFICATION_TAB_ADDED) {
+      content::WebContents* dest_content =
+          content::Details<content::WebContents>(details).ptr();
+      DCHECK(dest_content);
+      observer_list_.push_back(
+          new SafeBrowsingNavigationObserver(dest_content, this));
+      DCHECK(observer_list_.back());
+    } else if (type == chrome::NOTIFICATION_RETARGETING) {
+      RecordRetargeting(details);
+    }
+  }
+
+ protected:
+  ~TestNavigationObserverManager() override { observer_list_.clear(); }
+
+ private:
+  std::vector<SafeBrowsingNavigationObserver*> observer_list_;
+};
+
+class SBNavigationObserverBrowserTest : public InProcessBrowserTest {
+ public:
+  SBNavigationObserverBrowserTest() {}
+
+  void SetUpOnMainThread() override {
+    ASSERT_TRUE(embedded_test_server()->Start());
+    host_resolver()->AddRule("*", "127.0.0.1");
+    // Navigate to test page.
+    ui_test_utils::NavigateToURL(
+        browser(), embedded_test_server()->GetURL(kSingleFrameTestURL));
+    observer_manager_ = new TestNavigationObserverManager();
+    observer_ = new SafeBrowsingNavigationObserver(
+        browser()->tab_strip_model()->GetActiveWebContents(),
+        observer_manager_);
+    ASSERT_TRUE(observer_);
+    ASSERT_TRUE(InitialSetup());
+  }
+
+  bool InitialSetup() {
+    if (!browser())
+      return false;
+
+    if (!downloads_directory_.CreateUniqueTempDir())
+      return false;
+
+    // Set up default download path.
+    browser()->profile()->GetPrefs()->SetFilePath(
+        prefs::kDownloadDefaultDirectory, downloads_directory_.GetPath());
+    browser()->profile()->GetPrefs()->SetFilePath(
+        prefs::kSaveFileDefaultDirectory, downloads_directory_.GetPath());
+    browser()->profile()->GetPrefs()->SetBoolean(prefs::kPromptForDownload,
+                                                 false);
+    content::DownloadManager* manager =
+        content::BrowserContext::GetDownloadManager(browser()->profile());
+    DownloadPrefs::FromDownloadManager(manager)->ResetAutoOpen();
+    manager->RemoveAllDownloads();
+
+    return true;
+  }
+
+  void TearDownOnMainThread() override { delete observer_; }
+
+  // Most test cases will trigger downloads, though we don't really care if
+  // download completed or not. So we cancel downloads as soon as we record
+  // all the navigation event we need.

[Charlie Reis, 2016/10/25 04:52:27]

nit: events

[Jialiu Lin, 2016/10/27 02:15:20]

Done.

+  void CancelDownloads() {
+    std::vector<content::DownloadItem*> download_items;
+    content::DownloadManager* manager =
+        content::BrowserContext::GetDownloadManager(browser()->profile());
+    manager->GetAllDownloads(&download_items);
+    for (auto item : download_items)
+      item->Cancel(true);
+  }
+
+  // This function needs javascript support, only works on
+  // navigation_observer_tests.html and
+  // navigation_observer_multi_frame_tests.html.
+  void ClickTestLink(const char* test_name, int navigation_count) {
+    content::WebContents* main_web_contents =
+        browser()->tab_strip_model()->GetActiveWebContents();
+    std::string script = base::StringPrintf("clickLink('%s')", test_name);
+    EXPECT_TRUE(content::ExecuteScript(main_web_contents, script));
+    // Wait until all the navigation completes.

[Charlie Reis, 2016/10/25 04:52:26]

nit: navigations complete.

[Jialiu Lin, 2016/10/27 02:15:20]

Done.

+    for (int i = 0; i < navigation_count; i++) {
+      content::WebContents* active_contents =
+          browser()->tab_strip_model()->GetActiveWebContents();
+      content::WaitForLoadStopWithoutSuccessCheck(active_contents);
+    }
+    CancelDownloads();
+  }
+
+  void VerifyNavigationEvent(const GURL& expected_source_url,
+                             const GURL& expected_original_request_url,
+                             const GURL& expected_source_main_frame_url,
+                             bool expected_is_user_initiated,
+                             bool expected_has_committed,
+                             bool expected_has_server_redirect,
+                             const GURL& expected_destination_url,
+                             const NavigationEvent& actual_nav_event) {
+    EXPECT_EQ(expected_source_url, actual_nav_event.source_url);
+    EXPECT_EQ(expected_original_request_url,
+              actual_nav_event.original_request_url);
+    EXPECT_EQ(expected_source_main_frame_url,
+              actual_nav_event.source_main_frame_url);
+    EXPECT_EQ(expected_is_user_initiated, actual_nav_event.is_user_initiated);
+    EXPECT_EQ(expected_has_committed, actual_nav_event.has_committed);
+    EXPECT_EQ(expected_has_server_redirect,
+              actual_nav_event.has_server_redirect);
+    EXPECT_EQ(expected_destination_url, actual_nav_event.destination_url);
+  }
+
+  void VerifyHostToIpMap() {
+    // Since all testing pages have the same host, there is only one entry in
+    // host_to_ip_map_.
+    SafeBrowsingNavigationObserverManager::HostToIpMap* actual_host_ip_map =
+        host_to_ip_map();
+    ASSERT_EQ(std::size_t(1), actual_host_ip_map->size());
+    auto ip_list =
+        actual_host_ip_map->at(embedded_test_server()->base_url().host());
+    ASSERT_EQ(std::size_t(1), ip_list.size());
+    EXPECT_EQ(embedded_test_server()->host_port_pair().host(),
+              ip_list.back().ip);
+  }
+
+  SafeBrowsingNavigationObserverManager::NavigationMap* navigation_map() {
+    return observer_manager_->navigation_map();
+  }
+
+  SafeBrowsingNavigationObserverManager::HostToIpMap* host_to_ip_map() {
+    return observer_manager_->host_to_ip_map();
+  }
+
+ protected:
+  SafeBrowsingNavigationObserverManager* observer_manager_;
+  SafeBrowsingNavigationObserver* observer_;
+
+ private:
+  base::ScopedTempDir downloads_directory_;
+};
+
+// Click on a link and start download on the same page.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest, DirectDownload) {
+  ClickTestLink("direct_download", 1);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(1), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  // Since this test uses java script to mimic clicking on a link, there is no

[Charlie Reis, 2016/10/25 04:52:27]

nit: Javascript

Still, this sounds incorrect to me.  ClickTestLink uses ExecuteScript, which
boils down to ExecuteJavaScriptWithUserGestureForTests.  I would expect it to
have a user gesture, so I'm puzzled why it doesn't.

[Jialiu Lin, 2016/10/27 02:15:20]

Not sure why, but when I tested manually,
WebContentsObserver::DidGetUserInteraction() kicks in, but when used
ExecuteScript(), DidGetUserInteraction() never get triggered.

[Charlie Reis, 2016/10/28 23:36:08]

Hmm, I wonder if we have two different ways of tracking user gestures.  We
should mention this in the comment, since it's pretty surprising to me.  (I
suppose we could check with dominickn@, but it's nothing to delay this CL over.)

[Jialiu Lin, 2016/10/29 00:45:34]

It is probably because DidGetUserInteraction() only responses to input events
(a.k.a MouseDown, GestureScrollBegin,TouchStart, RawKeyDown), whereas
ExecuteJavaScriptWithUserGestureForTests() directly sets the has_user_gesture
bit in NavigationHandle.
I would say DidGetUserInteraction() is more reliable and it
prevents bad guys to use javascript to "click" around to evade.
https://cs.chromium.org/chromium/src/content/browser/renderer_host/render_wid...

I'll confirm with dominickn@ later. 

+  // actual user gesture, therefore is_user_initiated is false.
+  VerifyNavigationEvent(initial_url,   // source_url
+                        download_url,  // original_request_url
+                        initial_url,   // source_main_frame_url

[Charlie Reis, 2016/10/25 04:52:26]

Seems like this would be easier to read if the source_main_frame_url came before
original_request_url (similar to the declaration in NavigationEvent).

Similarly, it's probably useful for destination_url to come right after
original_request_url.

(Same in the unittest file.)

[Jialiu Lin, 2016/10/27 02:15:20]

Done.

[Charlie Reis, 2016/10/28 23:36:08]

Thanks!  I'm finding the tests much easier to read now.

+                        false,         // is_user_initiated,
+                        false,         // has_committed
+                        false,         // has_server_redirect
+                        download_url,  // destination_url
+                        nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Click on a link with rel="noreferrer" attribute, and start download on the
+// same page.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       DirectDownloadNoReferrer) {
+  ClickTestLink("direct_download_noreferrer", 1);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(1), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,   // source_url
+                        download_url,  // original_request_url
+                        initial_url,   // source_main_frame_url
+                        false,         // is_user_initiated,
+                        false,         // has_committed
+                        false,         // has_server_redirect
+                        download_url,  // destination_url
+                        nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Click on a link with rel="noreferrer" attribute, and start download in a
+// new tab using target=_blank.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       DirectDownloadNoReferrerTargetBlank) {
+  ClickTestLink("direct_download_noreferrer_target_blank", 1);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(1), nav_map->size());
+  ASSERT_EQ(std::size_t(2), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,   // source_url
+                        download_url,  // original_request_url
+                        initial_url,   // source_main_frame_url
+                        false,         // is_user_initiated,
+                        false,         // has_committed
+                        false,         // has_server_redirect
+                        download_url,  // destination_url
+                        nav_map->at(download_url).at(0));
+  VerifyNavigationEvent(download_url, download_url, download_url, false, false,

[Charlie Reis, 2016/10/25 04:52:27]

I think it would be useful to format this like the previous call, annotating
what each argument is.  (Same in the tests below.)

Can you also include a comment for why there are two items in the vector?  I'm
not sure what this second one is, and it seems to have lost the attribution to
initial_url.

[Jialiu Lin, 2016/10/27 02:15:20]

The first one comes from NOTIFICATION_RETARGETING, which indicates the actual
initiator. The second one is the true navigation event. 
So after we can obtain the true initiator info from NavigationHandle, the first
one will be gone. 
All the other retargeting related test cases (new tab download, target blank,
etc) have an extra NavigationEvent too due to the same reason.

I added a TODO to serve as a reminder. These tests need to be updated after
crbug.com/651895 is fixed.

[Charlie Reis, 2016/10/28 23:36:08]

Ah, that explains it.  Thanks!

+                        false, download_url, nav_map->at(download_url).at(1));
+  VerifyHostToIpMap();
+}
+
+// Click on a link which navigates to a page then redirect to download using

[Charlie Reis, 2016/10/25 04:52:26]

nit: redirects to a download

[Jialiu Lin, 2016/10/27 02:15:20]

Done.

+// META HTTP-EQUIV="refresh". All transitions happen in the same tab.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       SingleMetaRefreshRedirect) {
+  ClickTestLink("single_meta_refresh_redirect", 2);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL redirect_url = embedded_test_server()->GetURL(kRedirectURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(2), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(redirect_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,   // source_url
+                        redirect_url,  // original_request_url
+                        initial_url,   // source_main_frame_url
+                        false,         // is_user_initiated,
+                        true,          // has_committed
+                        false,         // has_server_redirect
+                        redirect_url,  // destination_url

[Charlie Reis, 2016/10/25 04:52:27]

This is a bit surprising when first reading the test, since you'd expect this to
be download_url at first glance.  It's probably worth mentioning in a comment
that client redirects commit and then generate a second NavigationEvent, unlike
server redirects.

(Are there any tests that show the server redirect behavior?  I don't see any. 
You can trigger one with "/server-redirect?dest_url" and the embedded test
server will make it work.)

[Jialiu Lin, 2016/10/27 02:15:20]

Sure. comment added. 
Server side redirection is tested in unittests by using SimulateRedirect()

[Charlie Reis, 2016/10/28 23:36:08]

We've found that unit tests (while they're useful for some things and certainly
fast) tend to not provide good coverage against regressions in practice.  I'd
suggest also including a browser test here for server redirects if possible,
since it shouldn't be too hard with the "/server-redirect?" URL syntax.

[Jialiu Lin, 2016/10/29 00:45:34]

Sure. redirect browser test added. 

+                        nav_map->at(redirect_url).at(0));
+  VerifyNavigationEvent(redirect_url, download_url, redirect_url, false, false,
+                        false, download_url, nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Click on a link which navigates to a page then redirect to download using

[Charlie Reis, 2016/10/25 04:52:27]

nit: redirects to a download

[Jialiu Lin, 2016/10/27 02:15:20]

Done.

+// META HTTP-EQUIV="refresh". First navigation happens in target blank.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       SingleMetaRefreshRedirectTargetBlank) {
+  ClickTestLink("single_meta_refresh_redirect_target_blank", 3);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL redirect_url = embedded_test_server()->GetURL(kRedirectURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(2), nav_map->size());
+  ASSERT_EQ(std::size_t(2), nav_map->at(redirect_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,   // source_url
+                        redirect_url,  // original_request_url
+                        initial_url,   // source_main_frame_url
+                        false,         // is_user_initiated,
+                        false,         // has_committed
+                        false,         // has_server_redirect
+                        redirect_url,  // destination_url
+                        nav_map->at(redirect_url).at(0));
+  VerifyNavigationEvent(redirect_url, redirect_url, redirect_url, false, true,
+                        false, redirect_url, nav_map->at(redirect_url).at(1));

[Charlie Reis, 2016/10/25 04:52:27]

What's going on in this one?  I'm not sure why we would be going from
redirect_url to redirect_url.  (Or is this because we set the source_url to the
destination_url when there is no source_url?  Maybe this would be clearer if the
source_url were empty in cases like this.)

[Jialiu Lin, 2016/10/27 02:15:20]

Yes, source should be empty after changes in observer.cc.

+  VerifyNavigationEvent(redirect_url, download_url, redirect_url, false, false,
+                        false, download_url, nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Click on a link which redirects twice before reaching download using
+// META HTTP-EQUIV="refresh". All transitions happen in the same tab.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       MultiMetaRefreshRedirects) {
+  ClickTestLink("multiple_meta_refresh_redirects", 3);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL first_redirect_url = embedded_test_server()->GetURL(
+      "/safe_browsing/download_protection/navigation_observer/"
+      "double_redirect.html");
+  GURL second_redirect_url = embedded_test_server()->GetURL(kRedirectURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(3), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(first_redirect_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(second_redirect_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,         // source_url
+                        first_redirect_url,  // original_request_url
+                        initial_url,         // source_main_frame_url
+                        false,               // is_user_initiated,
+                        true,                // has_committed
+                        false,               // has_server_redirect
+                        first_redirect_url,  // destination_url
+                        nav_map->at(first_redirect_url).at(0));
+  VerifyNavigationEvent(
+      first_redirect_url, second_redirect_url, first_redirect_url, false, true,
+      false, second_redirect_url, nav_map->at(second_redirect_url).at(0));
+  VerifyNavigationEvent(second_redirect_url, download_url, second_redirect_url,
+                        false, false, false, download_url,
+                        nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Click on a link which redirects to download using window.location.href.
+// All transitions happen in the same tab.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       WindowLocationHrefRedirect) {
+  ClickTestLink("window_location_href_redirect", 2);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(2), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(initial_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,  // source_url
+                        initial_url,  // original_request_url
+                        initial_url,  // source_main_frame_url
+                        false,        // is_user_initiated,
+                        true,         // has_committed
+                        false,        // has_server_redirect
+                        initial_url,  // destination_url
+                        nav_map->at(initial_url).at(0));
+  VerifyNavigationEvent(initial_url, download_url, initial_url, false, false,
+                        false, download_url, nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Click on a link which redirect twice until reaches download using a mixture

[Charlie Reis, 2016/10/25 04:52:26]

nit: redirects twice until it

[Jialiu Lin, 2016/10/27 02:15:20]

Done.

+// of meta refresh and window.location.href. All transitions happen in the same
+// tab.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest, MixRedirects) {
+  ClickTestLink("mix_redirects", 3);
+  GURL redirect_url = embedded_test_server()->GetURL(kRedirectURL);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(3), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(initial_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(redirect_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,  // source_url
+                        initial_url,  // original_request_url
+                        initial_url,  // source_main_frame_url
+                        false,        // is_user_initiated,
+                        true,         // has_committed
+                        false,        // has_server_redirect
+                        initial_url,  // destination_url
+                        nav_map->at(initial_url).at(0));
+  VerifyNavigationEvent(initial_url, redirect_url, initial_url, false, true,
+                        false, redirect_url, nav_map->at(redirect_url).at(0));
+  VerifyNavigationEvent(redirect_url, download_url, redirect_url, false, false,
+                        false, download_url, nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Use javascript to open download in a new tab.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest, NewTabDownload) {
+  ClickTestLink("new_tab_download", 4);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  GURL blank_url = GURL("about:blank");

[Charlie Reis, 2016/10/25 04:52:27]

nit: url::kAboutBlankURL

[Jialiu Lin, 2016/10/27 02:15:20]

Done.

+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(3), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(initial_url).size());
+  ASSERT_EQ(std::size_t(2), nav_map->at(blank_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,  // source_url
+                        initial_url,  // original_request_url
+                        initial_url,  // source_main_frame_url
+                        false,        // is_user_initiated,
+                        true,         // has_committed
+                        false,        // has_server_redirect
+                        initial_url,  // destination_url
+                        nav_map->at(initial_url).at(0));
+  VerifyNavigationEvent(initial_url, blank_url, initial_url, false, false,
+                        false, blank_url, nav_map->at(blank_url).at(0));
+  // Source and target are at different tabs.
+  EXPECT_NE(nav_map->at(blank_url).at(0).source_tab_id,
+            nav_map->at(blank_url).at(0).target_tab_id);
+  VerifyNavigationEvent(blank_url, blank_url, blank_url, false, false, false,
+                        blank_url, nav_map->at(blank_url).at(1));
+  EXPECT_EQ(nav_map->at(blank_url).at(1).source_tab_id,
+            nav_map->at(blank_url).at(1).target_tab_id);
+  VerifyNavigationEvent(blank_url, download_url, blank_url, false, false, false,
+                        download_url, nav_map->at(download_url).at(0));

[Charlie Reis, 2016/10/25 04:52:27]

Can you remind me how we're going to link this NavigationEvent to the one with
initial_url?  I forget what ties them together.

[Jialiu Lin, 2016/10/27 02:15:20]

Since all the NavigationEvents are keyed on their destination url, given a
download url (and its tab id),  look up NavigationMap to find corresponding
NavigationEvent, then search for the source_url  and source tab id as the new
destination url. Continue doing this until we reach a NavigationEvent initiated
by user or there is no more match in the map. 
In case NavigationEvent has empty source_url and does not initiated by user
(this rules out browser initiated navigation in a new tab), we then continue the
backward search using destination_url. If two NavigationEvents have the same
destination url (and tab id), then we pick the most recent one based on their
timestamps.

So in this specific case,  download_url -> blank_url->empty url
(blank_url)->initial_url

[Charlie Reis, 2016/10/28 23:36:08]

Thanks.  As discussed, the "most recent" approach has some downsides (e.g., it
might allow an attacker to partly misdirect blame to another URL he/she
controls), but I don't think it should be a problem if we just care about
finding URLs the attacker controls.

I'm curious-- that code (to generate the chain of events) will eventually live
in Chrome, right?  When we get around to adding it, we should make it the last
step in each of these tests, ensuring that it always gives us back the
initial_url.

[Jialiu Lin, 2016/10/29 00:45:34]

Yes, sure. That's my plan too.

+  EXPECT_EQ(nav_map->at(download_url).at(0).source_tab_id,
+            nav_map->at(download_url).at(0).target_tab_id);
+  VerifyHostToIpMap();
+}
+
+// Use javascript to open download in a new tab and download has a data url.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       NewTabDownloadWithDataURL) {
+  ClickTestLink("new_tab_download_with_data_url", 4);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL download_url = GURL(kDownloadDataURL);
+  GURL blank_url = GURL("about:blank");
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(3), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(initial_url).size());
+  ASSERT_EQ(std::size_t(2), nav_map->at(blank_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,  // source_url
+                        initial_url,  // original_request_url
+                        initial_url,  // source_main_frame_url
+                        false,        // is_user_initiated,
+                        true,         // has_committed
+                        false,        // has_server_redirect
+                        initial_url,  // destination_url
+                        nav_map->at(initial_url).at(0));
+  VerifyNavigationEvent(initial_url, blank_url, initial_url, false, false,
+                        false, blank_url, nav_map->at(blank_url).at(0));
+  // Source and target are at different tabs.
+  EXPECT_FALSE(nav_map->at(blank_url).at(0).source_tab_id ==
+               nav_map->at(blank_url).at(0).target_tab_id);
+  VerifyNavigationEvent(blank_url, blank_url, blank_url, false, false, false,
+                        blank_url, nav_map->at(blank_url).at(1));
+  EXPECT_TRUE(nav_map->at(blank_url).at(1).source_tab_id ==
+              nav_map->at(blank_url).at(1).target_tab_id);
+  VerifyNavigationEvent(blank_url, download_url, blank_url, false, false, false,
+                        download_url, nav_map->at(download_url).at(0));
+  EXPECT_TRUE(nav_map->at(download_url).at(0).source_tab_id ==
+              nav_map->at(download_url).at(0).target_tab_id);
+  // Since data url does does not have IP, host_to_ip_map_ should be empty.
+  EXPECT_EQ(std::size_t(0), host_to_ip_map()->size());
+}
+
+// TODO(jialiul): Need to figure out why this test is failing on Windows and
+// flacky on other platforms.

[Charlie Reis, 2016/10/25 04:52:27]

nit: flaky

[Jialiu Lin, 2016/10/27 02:15:20]

Done.

+#define MAYBE_DownloadViaHTML5FileApi DISABLED_DownloadViaHTML5FileApi
+// Download via html5 file API.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       MAYBE_DownloadViaHTML5FileApi) {
+  ClickTestLink("html5_file_api", 2);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  std::string download_url_str =
+      base::StringPrintf("filesystem:%stemporary/test.exe",
+                         embedded_test_server()->base_url().spec().c_str());
+  GURL download_url = GURL(download_url_str);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(2), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(initial_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,  // source_url
+                        initial_url,  // original_request_url
+                        initial_url,  // source_main_frame_url
+                        false,        // is_user_initiated,
+                        true,         // has_committed
+                        false,        // has_server_redirect
+                        initial_url,  // destination_url
+                        nav_map->at(initial_url).at(0));
+  VerifyNavigationEvent(initial_url, download_url, initial_url, false, false,
+                        false, download_url, nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Click a link in a subframe and start download.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       SubFrameDirectDownload) {
+  ui_test_utils::NavigateToURL(
+      browser(), embedded_test_server()->GetURL(kMultiFrameTestURL));
+  EXPECT_TRUE(content::WaitForLoadStop(
+      browser()->tab_strip_model()->GetActiveWebContents()));
+  std::string test_name =
+      base::StringPrintf("%s', '%s", "iframe1", "iframe_direct_download");
+  ClickTestLink(test_name.c_str(), 1);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL multi_frame_test_url =
+      embedded_test_server()->GetURL(kMultiFrameTestURL);
+  GURL iframe_url = embedded_test_server()->GetURL(kIframeDirectDownloadURL);
+  GURL iframe_retargeting_url =
+      embedded_test_server()->GetURL(kIframeRetargetingURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(4), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(multi_frame_test_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(iframe_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(iframe_retargeting_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,           // source_url
+                        multi_frame_test_url,  // original_request_url
+                        initial_url,           // source_main_frame_url
+                        true,                  // is_user_initiated,
+                        true,                  // has_committed
+                        false,                 // has_server_redirect
+                        multi_frame_test_url,  // destination_url
+                        nav_map->at(multi_frame_test_url).at(0));
+  VerifyNavigationEvent(iframe_url, iframe_url, multi_frame_test_url, false,
+                        true, false, iframe_url, nav_map->at(iframe_url).at(0));
+  VerifyNavigationEvent(iframe_retargeting_url, iframe_retargeting_url,
+                        multi_frame_test_url, false, true, false,
+                        iframe_retargeting_url,
+                        nav_map->at(iframe_retargeting_url).at(0));
+  VerifyNavigationEvent(iframe_url, download_url, multi_frame_test_url, false,
+                        false, false, download_url,
+                        nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// Click a link in a subframe and open download in a new tab.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest,
+                       SubFrameNewTabDownload) {
+  ui_test_utils::NavigateToURL(
+      browser(), embedded_test_server()->GetURL(kMultiFrameTestURL));
+  EXPECT_TRUE(content::WaitForLoadStop(
+      browser()->tab_strip_model()->GetActiveWebContents()));
+  std::string test_name =
+      base::StringPrintf("%s', '%s", "iframe2", "iframe_new_tab_download");
+  ClickTestLink(test_name.c_str(), 4);
+  GURL initial_url = embedded_test_server()->GetURL(kSingleFrameTestURL);
+  GURL multi_frame_test_url =
+      embedded_test_server()->GetURL(kMultiFrameTestURL);
+  GURL iframe_url = embedded_test_server()->GetURL(kIframeDirectDownloadURL);
+  GURL iframe_retargeting_url =
+      embedded_test_server()->GetURL(kIframeRetargetingURL);
+  GURL download_url = embedded_test_server()->GetURL(kDownloadItemURL);
+  GURL blank_url = GURL("about:blank");
+  auto nav_map = navigation_map();
+  ASSERT_TRUE(nav_map);
+  ASSERT_EQ(std::size_t(5), nav_map->size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(multi_frame_test_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(iframe_url).size());
+  ASSERT_EQ(std::size_t(2), nav_map->at(iframe_retargeting_url).size());
+  ASSERT_EQ(std::size_t(2), nav_map->at(blank_url).size());
+  ASSERT_EQ(std::size_t(1), nav_map->at(download_url).size());
+  VerifyNavigationEvent(initial_url,           // source_url
+                        multi_frame_test_url,  // original_request_url
+                        initial_url,           // source_main_frame_url
+                        true,                  // is_user_initiated,
+                        true,                  // has_committed
+                        false,                 // has_server_redirect
+                        multi_frame_test_url,  // destination_url
+                        nav_map->at(multi_frame_test_url).at(0));
+  VerifyNavigationEvent(iframe_url, iframe_url, multi_frame_test_url, false,
+                        true, false, iframe_url, nav_map->at(iframe_url).at(0));
+  VerifyNavigationEvent(iframe_retargeting_url, iframe_retargeting_url,
+                        multi_frame_test_url, false, true, false,
+                        iframe_retargeting_url,
+                        nav_map->at(iframe_retargeting_url).at(0));
+  VerifyNavigationEvent(iframe_retargeting_url, iframe_retargeting_url,
+                        multi_frame_test_url, false, true, false,
+                        iframe_retargeting_url,
+                        nav_map->at(iframe_retargeting_url).at(1));
+  VerifyNavigationEvent(iframe_retargeting_url, blank_url, multi_frame_test_url,
+                        false, false, false, blank_url,
+                        nav_map->at(blank_url).at(0));
+  VerifyNavigationEvent(blank_url, blank_url, blank_url, false, false, false,
+                        blank_url, nav_map->at(blank_url).at(1));
+  VerifyNavigationEvent(blank_url, download_url, blank_url, false, false, false,
+                        download_url, nav_map->at(download_url).at(0));
+  VerifyHostToIpMap();
+}
+
+// host_to_ip_map_ size should increase by one after a new navigation.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest, AddIPMapping) {
+  auto ip_map = host_to_ip_map();
+  std::string test_server_host(embedded_test_server()->base_url().host());
+  ip_map->insert(
+      std::make_pair(test_server_host, std::vector<ResolvedIPAddress>()));
+  ASSERT_EQ(std::size_t(0), ip_map->at(test_server_host).size());
+  ClickTestLink("direct_download", 1);
+  EXPECT_EQ(std::size_t(1), ip_map->at(test_server_host).size());
+  EXPECT_EQ(embedded_test_server()->host_port_pair().host(),
+            ip_map->at(test_server_host).back().ip);
+}
+
+// If we have already seen an IP associated with a host, update its timestamp.
+IN_PROC_BROWSER_TEST_F(SBNavigationObserverBrowserTest, IPListDedup) {
+  auto ip_map = host_to_ip_map();
+  std::string test_server_host(embedded_test_server()->base_url().host());
+  ip_map->insert(
+      std::make_pair(test_server_host, std::vector<ResolvedIPAddress>()));
+  base::Time yesterday(base::Time::Now() - base::TimeDelta::FromDays(1));
+  ip_map->at(test_server_host)
+      .push_back(ResolvedIPAddress(
+          yesterday, embedded_test_server()->host_port_pair().host()));
+  ASSERT_EQ(std::size_t(1), ip_map->at(test_server_host).size());
+  ClickTestLink("direct_download", 1);
+  EXPECT_EQ(std::size_t(1), ip_map->at(test_server_host).size());
+  EXPECT_EQ(embedded_test_server()->host_port_pair().host(),
+            ip_map->at(test_server_host).back().ip);
+  EXPECT_NE(yesterday, ip_map->at(test_server_host).front().timestamp);
+}
+
+}  // namespace safe_browsing