predictors: Add RedirectStatus histogram + fix redirects related bug.

chrome/browser/predictors/resource_prefetch_predictor_unittest.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/predictors/resource_prefetch_predictor.h"
 
 #include <iostream>
 #include <memory>
 #include <utility>
 
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/run_loop.h"
 #include "base/test/histogram_tester.h"
 #include "base/time/time.h"
 #include "chrome/browser/history/history_service_factory.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor_test_util.h"
 #include "chrome/test/base/testing_profile.h"
 #include "components/history/core/browser/history_service.h"
 #include "components/history/core/browser/history_types.h"
 #include "components/sessions/core/session_id.h"
 #include "content/public/browser/resource_request_info.h"
-#include "content/public/common/previews_state.h"
 #include "content/public/test/test_browser_thread_bundle.h"
 #include "net/http/http_response_headers.h"
 #include "net/url_request/url_request_context.h"
 #include "net/url_request/url_request_job.h"
 #include "net/url_request/url_request_test_util.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 using testing::ContainerEq;
 using testing::Pointee;
@@ skipping 182 matching lines @@
     return request;
   }
 
   void InitializePredictor() {
     predictor_->StartInitialization();
     base::RunLoop loop;
     loop.RunUntilIdle();  // Runs the DB lookup.
     profile_->BlockUntilHistoryProcessesPendingRequests();
   }
 
-  bool URLRequestSummaryAreEqual(const URLRequestSummary& lhs,
-                                 const URLRequestSummary& rhs) {
-    return lhs.navigation_id == rhs.navigation_id &&
-        lhs.resource_url == rhs.resource_url &&
-        lhs.resource_type == rhs.resource_type &&
-        lhs.mime_type == rhs.mime_type &&
-        lhs.was_cached == rhs.was_cached;
-  }
-
   void ResetPredictor() {
     ResourcePrefetchPredictorConfig config;
     config.max_urls_to_track = 3;
     config.max_hosts_to_track = 2;
     config.min_url_visit_count = 2;
     config.max_resources_per_entry = 4;
     config.max_consecutive_misses = 2;
     config.min_resource_confidence_to_trigger_prefetch = 0.5;
 
     // TODO(shishir): Enable the prefetching mode in the tests.
     config.mode |= ResourcePrefetchPredictorConfig::LEARNING;
     predictor_.reset(new ResourcePrefetchPredictor(config, profile_.get()));
     predictor_->set_mock_tables(mock_tables_);
   }
 
   void InitializeSampleData();
+  void TestRedirectStatusHistogram(
+      const std::string& predictor_initial_key,
+      const std::string& predictor_key,
+      const std::string& navigation_initial_url,
+      const std::string& navigation_url,
+      ResourcePrefetchPredictor::RedirectStatus expected_status);
 
   content::TestBrowserThreadBundle thread_bundle_;
   std::unique_ptr<TestingProfile> profile_;
   net::TestURLRequestContext url_request_context_;
 
   std::unique_ptr<ResourcePrefetchPredictor> predictor_;
-  scoped_refptr<StrictMock<MockResourcePrefetchPredictorTables> > mock_tables_;
+  scoped_refptr<StrictMock<MockResourcePrefetchPredictorTables>> mock_tables_;
 
   PrefetchDataMap test_url_data_;
   PrefetchDataMap test_host_data_;
   RedirectDataMap test_url_redirect_data_;
   RedirectDataMap test_host_redirect_data_;
   PrefetchData empty_resource_data_;
   RedirectData empty_redirect_data_;
 
   MockURLRequestJobFactory url_request_job_factory_;
   EmptyURLRequestDelegate url_request_delegate_;
@@ skipping 150 matching lines @@
     InitializeRedirectStat(microsoft.add_redirect_endpoints(),
                            "www.microsoft.com", 10, 0, 0);
 
     test_host_redirect_data_.clear();
     test_host_redirect_data_.insert(std::make_pair(bbc.primary_key(), bbc));
     test_host_redirect_data_.insert(
         std::make_pair(microsoft.primary_key(), microsoft));
   }
 }
 
+void ResourcePrefetchPredictorTest::TestRedirectStatusHistogram(
+    const std::string& predictor_initial_key,
+    const std::string& predictor_key,
+    const std::string& navigation_initial_url,
+    const std::string& navigation_url,
+    ResourcePrefetchPredictor::RedirectStatus expected_status) {
+  // Database initialization.
+  const std::string& script_url = "https://cdn.google.com/script.js";
+  PrefetchData google = CreatePrefetchData(predictor_key, 1);
+  // We need at least one resource for prediction.
+  InitializeResourceData(google.add_resources(), script_url,
+                         content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
+                         net::MEDIUM, false, false);
+  predictor_->host_table_cache_->insert(
+      std::make_pair(google.primary_key(), google));
+
+  if (predictor_initial_key != predictor_key) {
+    RedirectData redirect = CreateRedirectData(predictor_initial_key, 1);
+    InitializeRedirectStat(redirect.add_redirect_endpoints(), predictor_key, 10,
+                           0, 0);
+    predictor_->host_redirect_table_cache_->insert(
+        std::make_pair(redirect.primary_key(), redirect));
+  }
+
+  // Navigation simulation.
+  using testing::_;
+  EXPECT_CALL(*mock_tables_.get(), UpdateData(_, _, _, _))
+      .Times(testing::AtLeast(1));
+  URLRequestSummary initial =
+      CreateURLRequestSummary(1, navigation_initial_url);
+  predictor_->RecordURLRequest(initial);
+
+  if (navigation_initial_url != navigation_url) {
+    URLRequestSummary redirect =
+        CreateRedirectRequestSummary(1, navigation_initial_url, navigation_url);
+    predictor_->RecordURLRedirect(redirect);
+  }
+  NavigationID navigation_id = CreateNavigationID(1, navigation_url);
+
+  URLRequestSummary script = CreateURLRequestSummary(
+      1, navigation_url, script_url, content::RESOURCE_TYPE_SCRIPT);
+  predictor_->RecordURLResponse(script);
+
+  predictor_->RecordMainFrameLoadComplete(navigation_id);
+  profile_->BlockUntilHistoryProcessesPendingRequests();
+
+  // Histogram check.
+  histogram_tester_->ExpectBucketCount(
+      internal::kResourcePrefetchPredictorRedirectStatusHistogram,
+      static_cast<int>(expected_status), 1);
+}
+
 // Confirm that there's been no shift in the
 // ResourceData_Priority/net::RequestPriority equivalence.
 static_assert(static_cast<int>(net::MINIMUM_PRIORITY) ==
               static_cast<int>(
                   ResourceData_Priority_REQUEST_PRIORITY_THROTTLED),
               "Database/Net priority mismatch: Minimum");
 static_assert(static_cast<int>(net::MAXIMUM_PRIORITY) ==
               static_cast<int>(ResourceData_Priority_REQUEST_PRIORITY_HIGHEST),
               "Database/Net priority mismatch: Maximum");
 
@@ skipping 690 matching lines @@
   URLRequestSummary resource3 = CreateURLRequestSummary(
       1, "http://www.google.com", "http://google.com/script2.js",
       content::RESOURCE_TYPE_SCRIPT, net::MEDIUM, "text/javascript", false);
   predictor_->OnSubresourceResponse(resource1);
   predictor_->OnSubresourceResponse(resource2);
   predictor_->OnSubresourceResponse(resource3);
 
   EXPECT_EQ(1U, predictor_->inflight_navigations_.size());
   EXPECT_EQ(3U, predictor_->inflight_navigations_[main_frame1.navigation_id]
                     ->subresource_requests.size());
-  EXPECT_TRUE(URLRequestSummaryAreEqual(
-      resource1, predictor_->inflight_navigations_[main_frame1.navigation_id]
-                     ->subresource_requests[0]));
-  EXPECT_TRUE(URLRequestSummaryAreEqual(
-      resource2, predictor_->inflight_navigations_[main_frame1.navigation_id]
-                     ->subresource_requests[1]));
-  EXPECT_TRUE(URLRequestSummaryAreEqual(
-      resource3, predictor_->inflight_navigations_[main_frame1.navigation_id]
-                     ->subresource_requests[2]));
+  EXPECT_EQ(resource1,
+            predictor_->inflight_navigations_[main_frame1.navigation_id]
+                ->subresource_requests[0]);
+  EXPECT_EQ(resource2,
+            predictor_->inflight_navigations_[main_frame1.navigation_id]
+                ->subresource_requests[1]);
+  EXPECT_EQ(resource3,
+            predictor_->inflight_navigations_[main_frame1.navigation_id]
+                ->subresource_requests[2]);
 }
 
 TEST_F(ResourcePrefetchPredictorTest, HandledResourceTypes) {
   EXPECT_TRUE(ResourcePrefetchPredictor::IsHandledResourceType(
       content::RESOURCE_TYPE_STYLESHEET, "bogus/mime-type"));
   EXPECT_TRUE(ResourcePrefetchPredictor::IsHandledResourceType(
       content::RESOURCE_TYPE_STYLESHEET, ""));
   EXPECT_FALSE(ResourcePrefetchPredictor::IsHandledResourceType(
       content::RESOURCE_TYPE_WORKER, "text/css"));
   EXPECT_FALSE(ResourcePrefetchPredictor::IsHandledResourceType(
@@ skipping 308 matching lines @@
   EXPECT_EQ(redirect_endpoint, "https://google.com");
 
   EXPECT_FALSE(predictor_->GetRedirectEndpoint("http://fb.com", data_map,
                                                &redirect_endpoint));
   EXPECT_FALSE(predictor_->GetRedirectEndpoint("http://404.com", data_map,
                                                &redirect_endpoint));
 }
 
 TEST_F(ResourcePrefetchPredictorTest, GetPrefetchData) {
   const GURL main_frame_url("http://google.com/?query=cats");
-  std::vector<GURL> urls;
+  ResourcePrefetchPredictor::Prediction prediction;
+  std::vector<GURL>& urls = prediction.subresource_urls;
   // No prefetch data.
-  EXPECT_FALSE(predictor_->GetPrefetchData(main_frame_url, &urls));
+  EXPECT_FALSE(predictor_->GetPrefetchData(main_frame_url, &prediction));
 
   // Add a resource associated with the main frame host.
   PrefetchData google_host = CreatePrefetchData("google.com", 1);
   const std::string script_url = "https://cdn.google.com/script.js";
   InitializeResourceData(google_host.add_resources(), script_url,
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert(
       std::make_pair(google_host.primary_key(), google_host));
 
   urls.clear();
-  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &urls));
+  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(script_url)));
 
   // Add a resource associated with the main frame url.
   PrefetchData google_url =
       CreatePrefetchData("http://google.com/?query=cats", 2);
   const std::string image_url = "https://cdn.google.com/image.png";
   InitializeResourceData(google_url.add_resources(), image_url,
                          content::RESOURCE_TYPE_IMAGE, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->url_table_cache_->insert(
       std::make_pair(google_url.primary_key(), google_url));
 
   urls.clear();
-  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &urls));
+  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(image_url)));
 
   // Add host-based redirect.
   RedirectData host_redirect = CreateRedirectData("google.com", 3);
   InitializeRedirectStat(host_redirect.add_redirect_endpoints(),
                          "www.google.com", 10, 0, 0);
   predictor_->host_redirect_table_cache_->insert(
       std::make_pair(host_redirect.primary_key(), host_redirect));
 
   // Nothing changed: new redirect endpoint doesn't have any associated
   // resources
   urls.clear();
-  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &urls));
+  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(image_url)));
 
   // Add a resource associated with host redirect endpoint.
   PrefetchData www_google_host = CreatePrefetchData("www.google.com", 4);
   const std::string style_url = "https://cdn.google.com/style.css";
   InitializeResourceData(www_google_host.add_resources(), style_url,
                          content::RESOURCE_TYPE_STYLESHEET, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert(
       std::make_pair(www_google_host.primary_key(), www_google_host));
 
   urls.clear();
-  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &urls));
+  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(style_url)));
 
   // Add url-based redirect.
   RedirectData url_redirect =
       CreateRedirectData("http://google.com/?query=cats", 5);
   InitializeRedirectStat(url_redirect.add_redirect_endpoints(),
                          "https://www.google.com/?query=cats", 10, 0, 0);
   predictor_->url_redirect_table_cache_->insert(
       std::make_pair(url_redirect.primary_key(), url_redirect));
 
   // Url redirect endpoint doesn't have associated resources.
   urls.clear();
-  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &urls));
+  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(style_url)));
 
   // Add a resource associated with url redirect endpoint.
   PrefetchData www_google_url =
       CreatePrefetchData("https://www.google.com/?query=cats", 4);
   const std::string font_url = "https://cdn.google.com/comic-sans-ms.woff";
   InitializeResourceData(www_google_url.add_resources(), font_url,
                          content::RESOURCE_TYPE_FONT_RESOURCE, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->url_table_cache_->insert(
       std::make_pair(www_google_url.primary_key(), www_google_url));
 
   urls.clear();
-  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &urls));
+  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(font_url)));
 }
 
 TEST_F(ResourcePrefetchPredictorTest, TestPrecisionRecallHistograms) {
   using testing::_;
   EXPECT_CALL(*mock_tables_.get(), UpdateData(_, _, _, _));
 
   // Fill the database with 3 resources: 1 useful, 2 useless.
   const std::string main_frame_url = "http://google.com/?query=cats";
   PrefetchData google = CreatePrefetchData("google.com", 1);
 
   const std::string script_url = "https://cdn.google.com/script.js";
   InitializeResourceData(google.add_resources(), script_url,
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   InitializeResourceData(google.add_resources(), script_url + "foo",
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   InitializeResourceData(google.add_resources(), script_url + "bar",
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert(
       std::make_pair(google.primary_key(), google));
 
-  std::vector<GURL> urls;
-  EXPECT_TRUE(predictor_->GetPrefetchData(GURL(main_frame_url), &urls));
+  ResourcePrefetchPredictor::Prediction prediction;
+  EXPECT_TRUE(predictor_->GetPrefetchData(GURL(main_frame_url), &prediction));
 
   // Simulate a navigation with 2 resources, one we know, one we don't.
   URLRequestSummary main_frame = CreateURLRequestSummary(1, main_frame_url);
   predictor_->RecordURLRequest(main_frame);
 
   URLRequestSummary script = CreateURLRequestSummary(
       1, main_frame_url, script_url, content::RESOURCE_TYPE_SCRIPT);
   predictor_->RecordURLResponse(script);
 
   URLRequestSummary new_script = CreateURLRequestSummary(
       1, main_frame_url, script_url + "2", content::RESOURCE_TYPE_SCRIPT);
   predictor_->RecordURLResponse(new_script);
 
   predictor_->RecordMainFrameLoadComplete(main_frame.navigation_id);
   profile_->BlockUntilHistoryProcessesPendingRequests();
 
   histogram_tester_->ExpectBucketCount(
       internal::kResourcePrefetchPredictorRecallHistogram, 50, 1);
   histogram_tester_->ExpectBucketCount(
       internal::kResourcePrefetchPredictorPrecisionHistogram, 33, 1);
   histogram_tester_->ExpectBucketCount(
       internal::kResourcePrefetchPredictorCountHistogram, 3, 1);
 }
 
+TEST_F(ResourcePrefetchPredictorTest, TestRedirectStatusNoRedirect) {
+  TestRedirectStatusHistogram(
+      "google.com", "google.com", "http://google.com?query=cats",
+      "http://google.com?query=cats",
+      ResourcePrefetchPredictor::RedirectStatus::NO_REDIRECT);
+}
+
+TEST_F(ResourcePrefetchPredictorTest,
+       TestRedirectStatusNoRedirectButPredicted) {
+  TestRedirectStatusHistogram(
+      "google.com", "www.google.com", "http://google.com?query=cats",
+      "http://google.com?query=cats",
+      ResourcePrefetchPredictor::RedirectStatus::NO_REDIRECT_BUT_PREDICTED);
+}
+
+TEST_F(ResourcePrefetchPredictorTest, TestRedirectStatusRedirectNotPredicted) {
+  TestRedirectStatusHistogram(
+      "google.com", "google.com", "http://google.com?query=cats",
+      "http://www.google.com?query=cats",
+      ResourcePrefetchPredictor::RedirectStatus::REDIRECT_NOT_PREDICTED);
+}
+
+TEST_F(ResourcePrefetchPredictorTest,
+       TestRedirectStatusRedirectWrongPredicted) {
+  TestRedirectStatusHistogram(
+      "google.com", "google.fr", "http://google.com?query=cats",
+      "http://www.google.com?query=cats",
+      ResourcePrefetchPredictor::RedirectStatus::REDIRECT_WRONG_PREDICTED);
+}
+
+TEST_F(ResourcePrefetchPredictorTest,
+       TestRedirectStatusRedirectCorrectlyPredicted) {
+  TestRedirectStatusHistogram(
+      "google.com", "www.google.com", "http://google.com?query=cats",
+      "http://www.google.com?query=cats",
+      ResourcePrefetchPredictor::RedirectStatus::REDIRECT_CORRECTLY_PREDICTED);
+}
+
 TEST_F(ResourcePrefetchPredictorTest, TestPrefetchingDurationHistogram) {
   // Prefetching duration for an url without resources in the database
   // shouldn't be recorded.
   const std::string main_frame_url = "http://google.com/?query=cats";
   predictor_->StartPrefetching(GURL(main_frame_url), PrefetchOrigin::EXTERNAL);
   predictor_->StopPrefetching(GURL(main_frame_url));
   histogram_tester_->ExpectTotalCount(
       internal::kResourcePrefetchPredictorPrefetchingDurationHistogram, 0);
 
   // Fill the database to record a duration.
   PrefetchData google = CreatePrefetchData("google.com", 1);
   InitializeResourceData(
       google.add_resources(), "https://cdn.google.com/script.js",
       content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1, net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert(
       std::make_pair(google.primary_key(), google));
 
   predictor_->StartPrefetching(GURL(main_frame_url), PrefetchOrigin::EXTERNAL);
   predictor_->StopPrefetching(GURL(main_frame_url));
   histogram_tester_->ExpectTotalCount(
       internal::kResourcePrefetchPredictorPrefetchingDurationHistogram, 1);
 }
 
 }  // namespace predictors