Omnibox - Boost Frequency Scores Based on Number of Matching Pages

components/omnibox/browser/scored_history_match_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/omnibox/browser/scored_history_match.h"
 
 #include <algorithm>
 #include <memory>
 
 #include "base/auto_reset.h"
 #include "base/bind.h"
 #include "base/i18n/break_iterator.h"
 #include "base/memory/ptr_util.h"
 #include "base/strings/string16.h"
 #include "base/strings/utf_string_conversions.h"
+#include "components/omnibox/browser/omnibox_field_trial.h"
 #include "components/search_engines/search_terms_data.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 using base::ASCIIToUTF16;
 using testing::ElementsAre;
 using testing::Pair;
 
 namespace {
 
@@ skipping 89 matching lines @@
     // Find the first word start.
     while (iter.Advance() && !iter.IsWord()) {
     }
     term_word_starts[0] = iter.prev();
   }
   RowWordStarts row_word_starts;
   String16SetFromString16(url, &row_word_starts.url_word_starts_);
   String16SetFromString16(title, &row_word_starts.title_word_starts_);
   ScoredHistoryMatch scored_match(history::URLRow(GURL(url)), VisitInfoVector(),
                                   term, term_vector, term_word_starts,
-                                  row_word_starts, false, base::Time::Max());
+                                  row_word_starts, false, 1, base::Time::Max());
   scored_match.url_matches = MatchTermInString(term, url, 0);
   scored_match.title_matches = MatchTermInString(term, title, 0);
   scored_match.topicality_threshold_ = -1;
   return scored_match.GetTopicalityScore(1, url, term_word_starts,
                                          row_word_starts);
 }
 
 TEST_F(ScoredHistoryMatchTest, Scoring) {
   // We use NowFromSystemTime() because MakeURLRow uses the same function
   // to calculate last visit time when building a row.
   base::Time now = base::Time::NowFromSystemTime();
 
   history::URLRow row_a(MakeURLRow("http://fedcba", "abcd bcd", 3, 30, 1));
   RowWordStarts word_starts_a;
   PopulateWordStarts(row_a, &word_starts_a);
   WordStarts one_word_no_offset(1, 0u);
   VisitInfoVector visits_a = CreateVisitInfoVector(3, 30, now);
   // Mark one visit as typed.
   visits_a[0].second = ui::PAGE_TRANSITION_TYPED;
   ScoredHistoryMatch scored_a(row_a, visits_a, ASCIIToUTF16("abc"),
                               Make1Term("abc"), one_word_no_offset,
-                              word_starts_a, false, now);
+                              word_starts_a, false, 1, now);
 
   // Test scores based on visit_count.
   history::URLRow row_b(MakeURLRow("http://abcdef", "abcd bcd", 10, 30, 1));
   RowWordStarts word_starts_b;
   PopulateWordStarts(row_b, &word_starts_b);
   VisitInfoVector visits_b = CreateVisitInfoVector(10, 30, now);
   visits_b[0].second = ui::PAGE_TRANSITION_TYPED;
   ScoredHistoryMatch scored_b(row_b, visits_b, ASCIIToUTF16("abc"),
                               Make1Term("abc"), one_word_no_offset,
-                              word_starts_b, false, now);
+                              word_starts_b, false, 1, now);
   EXPECT_GT(scored_b.raw_score, scored_a.raw_score);
 
   // Test scores based on last_visit.
   history::URLRow row_c(MakeURLRow("http://abcdef", "abcd bcd", 3, 10, 1));
   RowWordStarts word_starts_c;
   PopulateWordStarts(row_c, &word_starts_c);
   VisitInfoVector visits_c = CreateVisitInfoVector(3, 10, now);
   visits_c[0].second = ui::PAGE_TRANSITION_TYPED;
   ScoredHistoryMatch scored_c(row_c, visits_c, ASCIIToUTF16("abc"),
                               Make1Term("abc"), one_word_no_offset,
-                              word_starts_c, false, now);
+                              word_starts_c, false, 1, now);
   EXPECT_GT(scored_c.raw_score, scored_a.raw_score);
 
   // Test scores based on typed_count.
   history::URLRow row_d(MakeURLRow("http://abcdef", "abcd bcd", 3, 30, 3));
   RowWordStarts word_starts_d;
   PopulateWordStarts(row_d, &word_starts_d);
   VisitInfoVector visits_d = CreateVisitInfoVector(3, 30, now);
   visits_d[0].second = ui::PAGE_TRANSITION_TYPED;
   visits_d[1].second = ui::PAGE_TRANSITION_TYPED;
   visits_d[2].second = ui::PAGE_TRANSITION_TYPED;
   ScoredHistoryMatch scored_d(row_d, visits_d, ASCIIToUTF16("abc"),
                               Make1Term("abc"), one_word_no_offset,
-                              word_starts_d, false, now);
+                              word_starts_d, false, 1, now);
   EXPECT_GT(scored_d.raw_score, scored_a.raw_score);
 
   // Test scores based on a terms appearing multiple times.
   history::URLRow row_e(MakeURLRow(
       "http://csi.csi.csi/csi_csi",
       "CSI Guide to CSI Las Vegas, CSI New York, CSI Provo", 3, 30, 3));
   RowWordStarts word_starts_e;
   PopulateWordStarts(row_e, &word_starts_e);
   const VisitInfoVector visits_e = visits_d;
   ScoredHistoryMatch scored_e(row_e, visits_e, ASCIIToUTF16("csi"),
                               Make1Term("csi"), one_word_no_offset,
-                              word_starts_e, false, now);
+                              word_starts_e, false, 1, now);
   EXPECT_LT(scored_e.raw_score, 1400);
 
   // Test that a result with only a mid-term match (i.e., not at a word
   // boundary) scores 0.
   ScoredHistoryMatch scored_f(row_a, visits_a, ASCIIToUTF16("cd"),
                               Make1Term("cd"), one_word_no_offset,
-                              word_starts_a, false, now);
+                              word_starts_a, false, 1, now);
   EXPECT_EQ(scored_f.raw_score, 0);
 }
 
 TEST_F(ScoredHistoryMatchTest, ScoringBookmarks) {
   // We use NowFromSystemTime() because MakeURLRow uses the same function
   // to calculate last visit time when building a row.
   base::Time now = base::Time::NowFromSystemTime();
 
   std::string url_string("http://fedcba");
   const GURL url(url_string);
   history::URLRow row(MakeURLRow(url_string.c_str(), "abcd bcd", 8, 3, 1));
   RowWordStarts word_starts;
   PopulateWordStarts(row, &word_starts);
   WordStarts one_word_no_offset(1, 0u);
   VisitInfoVector visits = CreateVisitInfoVector(8, 3, now);
   ScoredHistoryMatch scored(row, visits, ASCIIToUTF16("abc"), Make1Term("abc"),
-                            one_word_no_offset, word_starts, false, now);
+                            one_word_no_offset, word_starts, false, 1, now);
   // Now check that if URL is bookmarked then its score increases.
   base::AutoReset<float> reset(&ScoredHistoryMatch::bookmark_value_, 5);
   ScoredHistoryMatch scored_with_bookmark(row, visits, ASCIIToUTF16("abc"),
                                           Make1Term("abc"), one_word_no_offset,
-                                          word_starts, true, now);
+                                          word_starts, true, 1, now);
   EXPECT_GT(scored_with_bookmark.raw_score, scored.raw_score);
 }
 
 TEST_F(ScoredHistoryMatchTest, ScoringTLD) {
   // We use NowFromSystemTime() because MakeURLRow uses the same function
   // to calculate last visit time when building a row.
   base::Time now = base::Time::NowFromSystemTime();
 
   // By default the URL should not be returned for a query that includes "com".
   std::string url_string("http://fedcba.com/");
   const GURL url(url_string);
   history::URLRow row(MakeURLRow(url_string.c_str(), "", 8, 3, 1));
   RowWordStarts word_starts;
   PopulateWordStarts(row, &word_starts);
   WordStarts two_words_no_offsets(2, 0u);
   VisitInfoVector visits = CreateVisitInfoVector(8, 3, now);
   ScoredHistoryMatch scored(row, visits, ASCIIToUTF16("fed com"),
                             Make2Terms("fed", "com"), two_words_no_offsets,
-                            word_starts, false, now);
+                            word_starts, false, 1, now);
   EXPECT_EQ(0, scored.raw_score);
 
   // Now allow credit for the match in the TLD.
   base::AutoReset<bool> reset(&ScoredHistoryMatch::allow_tld_matches_, true);
   ScoredHistoryMatch scored_with_tld(
       row, visits, ASCIIToUTF16("fed com"), Make2Terms("fed", "com"),
-      two_words_no_offsets, word_starts, false, now);
+      two_words_no_offsets, word_starts, false, 1, now);
   EXPECT_GT(scored_with_tld.raw_score, 0);
 }
 
 TEST_F(ScoredHistoryMatchTest, ScoringScheme) {
   // We use NowFromSystemTime() because MakeURLRow uses the same function
   // to calculate last visit time when building a row.
   base::Time now = base::Time::NowFromSystemTime();
 
   // By default the URL should not be returned for a query that includes "http".
   std::string url_string("http://fedcba/");
   const GURL url(url_string);
   history::URLRow row(MakeURLRow(url_string.c_str(), "", 8, 3, 1));
   RowWordStarts word_starts;
   PopulateWordStarts(row, &word_starts);
   WordStarts two_words_no_offsets(2, 0u);
   VisitInfoVector visits = CreateVisitInfoVector(8, 3, now);
   ScoredHistoryMatch scored(row, visits, ASCIIToUTF16("fed http"),
                             Make2Terms("fed", "http"), two_words_no_offsets,
-                            word_starts, false, now);
+                            word_starts, false, 1, now);
   EXPECT_EQ(0, scored.raw_score);
 
   // Now allow credit for the match in the scheme.
   base::AutoReset<bool> reset(&ScoredHistoryMatch::allow_scheme_matches_, true);
   ScoredHistoryMatch scored_with_scheme(
       row, visits, ASCIIToUTF16("fed http"), Make2Terms("fed", "http"),
-      two_words_no_offsets, word_starts, false, now);
+      two_words_no_offsets, word_starts, false, 1, now);
   EXPECT_GT(scored_with_scheme.raw_score, 0);
 }
 
 TEST_F(ScoredHistoryMatchTest, Inlining) {
   // We use NowFromSystemTime() because MakeURLRow uses the same function
   // to calculate last visit time when building a row.
   base::Time now = base::Time::NowFromSystemTime();
   RowWordStarts word_starts;
   WordStarts one_word_no_offset(1, 0u);
   VisitInfoVector visits;
 
   {
     history::URLRow row(
         MakeURLRow("http://www.google.com", "abcdef", 3, 30, 1));
     PopulateWordStarts(row, &word_starts);
     ScoredHistoryMatch scored_a(row, visits, ASCIIToUTF16("g"), Make1Term("g"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_a.match_in_scheme);
     ScoredHistoryMatch scored_b(row, visits, ASCIIToUTF16("w"), Make1Term("w"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_b.match_in_scheme);
     ScoredHistoryMatch scored_c(row, visits, ASCIIToUTF16("h"), Make1Term("h"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_TRUE(scored_c.match_in_scheme);
     ScoredHistoryMatch scored_d(row, visits, ASCIIToUTF16("o"), Make1Term("o"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_d.match_in_scheme);
   }
 
   {
     history::URLRow row(MakeURLRow("http://teams.foo.com", "abcdef", 3, 30, 1));
     PopulateWordStarts(row, &word_starts);
     ScoredHistoryMatch scored_a(row, visits, ASCIIToUTF16("t"), Make1Term("t"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_a.match_in_scheme);
     ScoredHistoryMatch scored_b(row, visits, ASCIIToUTF16("f"), Make1Term("f"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_b.match_in_scheme);
     ScoredHistoryMatch scored_c(row, visits, ASCIIToUTF16("o"), Make1Term("o"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_c.match_in_scheme);
   }
 
   {
     history::URLRow row(
         MakeURLRow("https://www.testing.com", "abcdef", 3, 30, 1));
     PopulateWordStarts(row, &word_starts);
     ScoredHistoryMatch scored_a(row, visits, ASCIIToUTF16("t"), Make1Term("t"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_a.match_in_scheme);
     ScoredHistoryMatch scored_b(row, visits, ASCIIToUTF16("h"), Make1Term("h"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_TRUE(scored_b.match_in_scheme);
     ScoredHistoryMatch scored_c(row, visits, ASCIIToUTF16("w"), Make1Term("w"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_c.match_in_scheme);
   }
 
   {
     history::URLRow row(
         MakeURLRow("http://www.xn--1lq90ic7f1rc.cn/xnblah", "abcd", 3, 30, 1));
     PopulateWordStarts(row, &word_starts);
     ScoredHistoryMatch scored_a(row, visits, ASCIIToUTF16("x"), Make1Term("x"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_a.match_in_scheme);
     ScoredHistoryMatch scored_b(row, visits, ASCIIToUTF16("xn"),
                                 Make1Term("xn"), one_word_no_offset,
-                                word_starts, false, now);
+                                word_starts, false, 1, now);
     EXPECT_FALSE(scored_b.match_in_scheme);
     ScoredHistoryMatch scored_c(row, visits, ASCIIToUTF16("w"), Make1Term("w"),
-                                one_word_no_offset, word_starts, false, now);
+                                one_word_no_offset, word_starts, false, 1, now);
     EXPECT_FALSE(scored_c.match_in_scheme);
   }
 }
 
 TEST_F(ScoredHistoryMatchTest, GetTopicalityScoreTrailingSlash) {
   const float hostname = GetTopicalityScoreOfTermAgainstURLAndTitle(
       ASCIIToUTF16("def"), ASCIIToUTF16("http://abc.def.com/"),
       ASCIIToUTF16("Non-Matching Title"));
   const float hostname_no_slash = GetTopicalityScoreOfTermAgainstURLAndTitle(
       ASCIIToUTF16("def"), ASCIIToUTF16("http://abc.def.com"),
@@ skipping 118 matching lines @@
 }
 
 TEST_F(ScoredHistoryMatchTest, GetFrequency) {
   // Build a fake ScoredHistoryMatch, which we'll then reuse multiple times.
   history::URLRow row(GURL("http://foo"));
   RowWordStarts row_word_starts;
   PopulateWordStarts(row, &row_word_starts);
   base::Time now(base::Time::Max());
   VisitInfoVector visits;
   ScoredHistoryMatch match(row, visits, ASCIIToUTF16("foo"), Make1Term("foo"),
-                           WordStarts{0}, row_word_starts, false, now);
+                           WordStarts{0}, row_word_starts, false, 1, now);
 
   // Record the score for one untyped visit.
   visits = {{now, ui::PAGE_TRANSITION_LINK}};
   const float one_untyped_score = match.GetFrequency(now, false, visits);
 
   // The score for one typed visit should be larger.
   visits = VisitInfoVector{{now, ui::PAGE_TRANSITION_TYPED}};
   const float one_typed_score = match.GetFrequency(now, false, visits);
   EXPECT_GT(one_typed_score, one_untyped_score);
 
@@ skipping 79 matching lines @@
     EXPECT_EQ(two_visits_score_uses_sum,
               match.GetFrequency(now, false, visits));
 
     // Check again with the third visit being typed.
     visits[2].second = ui::PAGE_TRANSITION_TYPED;
     EXPECT_EQ(two_visits_score_uses_sum,
               match.GetFrequency(now, false, visits));
   }
 }
 
+TEST_F(ScoredHistoryMatchTest, GetDocumentSpecificityScore) {
+  // Build a fake ScoredHistoryMatch, which we'll then reuse multiple times.
+  history::URLRow row(GURL("http://foo"));
+  RowWordStarts row_word_starts;
+  PopulateWordStarts(row, &row_word_starts);
+  base::Time now(base::Time::Max());
+  VisitInfoVector visits;
+  ScoredHistoryMatch match(row, visits, ASCIIToUTF16("foo"), Make1Term("foo"),
+                           WordStarts{0}, row_word_starts, false, 1, now);
+
+  EXPECT_EQ(1.0, match.GetDocumentSpecificityScore(1));
+  EXPECT_EQ(1.0, match.GetDocumentSpecificityScore(5));
+  EXPECT_EQ(1.0, match.GetDocumentSpecificityScore(50));
+
+  OmniboxFieldTrial::NumMatchesScores num_matches_to_document_specificity_score;

[Peter Kasting, 2016/12/10 02:22:26]

Nit: Might want a blank line here since the lack of one implies this variable is
only used for this block.

[Mark P, 2016/12/11 05:11:37]

Done.

+  {
+    num_matches_to_document_specificity_score = {{1, 3.0}};
+    base::AutoReset<OmniboxFieldTrial::NumMatchesScores*> tmp(

[Peter Kasting, 2016/12/10 02:22:26]

Nit: If you're going to declare the override outside these blocks, might as well
hoist the AutoReset object there too.  Then you only need it once and the blocks
are maximally readable.

[Mark P, 2016/12/11 05:11:37]

Done.

(I forgot that I could modify the replacement variable multiple times and have
everything work.)

+        &ScoredHistoryMatch::
+            num_matches_to_document_specificity_score_override_,
+        &num_matches_to_document_specificity_score);
+    EXPECT_EQ(3.0, match.GetDocumentSpecificityScore(1));
+    EXPECT_EQ(1.0, match.GetDocumentSpecificityScore(5));
+  }
+
+  {
+    num_matches_to_document_specificity_score = {{1, 3.0}, {3, 1.5}};
+    base::AutoReset<OmniboxFieldTrial::NumMatchesScores*> tmp(
+        &ScoredHistoryMatch::
+            num_matches_to_document_specificity_score_override_,
+        &num_matches_to_document_specificity_score);
+    EXPECT_EQ(3.0, match.GetDocumentSpecificityScore(1));
+    EXPECT_EQ(1.5, match.GetDocumentSpecificityScore(2));
+    EXPECT_EQ(1.5, match.GetDocumentSpecificityScore(3));
+    EXPECT_EQ(1.0, match.GetDocumentSpecificityScore(4));
+  }
+}
+
 // This function only tests scoring of single terms that match exactly
 // once somewhere in the URL or title.
 TEST_F(ScoredHistoryMatchTest, GetTopicalityScore) {
   base::string16 url = ASCIIToUTF16(
       "http://abc.def.com/path1/path2?"
       "arg1=val1&arg2=val2#hash_component");
   base::string16 title = ASCIIToUTF16("here is a title");
   auto Score = [&](const char* term) {
     return GetTopicalityScoreOfTermAgainstURLAndTitle(ASCIIToUTF16(term), url,
                                                       title);
@@ skipping 56 matching lines @@
   // hqp_relevance_buckets = "0.0:100,1.0:200,4.0:500,8.0:900,10.0:1000";
   std::vector<ScoredHistoryMatch::ScoreMaxRelevance> hqp_buckets;
   hqp_buckets.push_back(std::make_pair(0.0, 100));
   hqp_buckets.push_back(std::make_pair(1.0, 200));
   hqp_buckets.push_back(std::make_pair(4.0, 500));
   hqp_buckets.push_back(std::make_pair(8.0, 900));
   hqp_buckets.push_back(std::make_pair(10.0, 1000));
   // Check when topicality score is zero.
   float topicality_score = 0.0;
   float frequency_score = 10.0;
-  // intermediate_score = 0.0 * 10.0 = 0.0.
+  float specificity_score = 1.0;
+  // intermediate_score = 0.0 * 10.0 * 1.0 = 0.0.
   EXPECT_EQ(0, ScoredHistoryMatch::GetFinalRelevancyScore(
-                   topicality_score, frequency_score, hqp_buckets));
+                   topicality_score, frequency_score, specificity_score,
+                   hqp_buckets));
 
   // Check when intermediate score falls at the border range.
   topicality_score = 0.4f;
   frequency_score = 10.0f;
-  // intermediate_score = 0.5 * 10.0 = 4.0.
+  // intermediate_score = 0.4 * 10.0 * 1.0 = 4.0.
   EXPECT_EQ(500, ScoredHistoryMatch::GetFinalRelevancyScore(
-                     topicality_score, frequency_score, hqp_buckets));
+                     topicality_score, frequency_score, specificity_score,
+                     hqp_buckets));
 
   // Checking the score that falls into one of the buckets.
   topicality_score = 0.5f;
   frequency_score = 10.0f;
-  // intermediate_score = 0.5 * 10.0 = 5.0.
-  EXPECT_EQ(600,  // 500 + (((900 - 500)/(8 -4)) * 1) = 600.
-            ScoredHistoryMatch::GetFinalRelevancyScore(
-                topicality_score, frequency_score, hqp_buckets));
+  // intermediate_score = 0.5 * 10.0 * 1.0 = 5.0.
+  EXPECT_EQ(
+      600,  // 500 + (((900 - 500)/(8 -4)) * 1) = 600.
+      ScoredHistoryMatch::GetFinalRelevancyScore(
+          topicality_score, frequency_score, specificity_score, hqp_buckets));
 
   // Never give the score greater than maximum specified.
   topicality_score = 0.5f;
   frequency_score = 22.0f;
-  // intermediate_score = 0.5 * 22.0 = 11.0
+  // intermediate_score = 0.5 * 22.0 * 1.0 = 11.0
   EXPECT_EQ(1000, ScoredHistoryMatch::GetFinalRelevancyScore(
-                      topicality_score, frequency_score, hqp_buckets));
+                      topicality_score, frequency_score, specificity_score,
+                      hqp_buckets));
 }
 
 // Test the function GetHQPBucketsFromString().
 TEST_F(ScoredHistoryMatchTest, GetHQPBucketsFromString) {
   std::string buckets_str = "0.0:400,1.5:600,12.0:1300,20.0:1399";
   std::vector<ScoredHistoryMatch::ScoreMaxRelevance> hqp_buckets;
 
   EXPECT_TRUE(
       ScoredHistoryMatch::GetHQPBucketsFromString(buckets_str, &hqp_buckets));
   EXPECT_THAT(hqp_buckets, ElementsAre(Pair(0.0, 400), Pair(1.5, 600),
                                        Pair(12.0, 1300), Pair(20.0, 1399)));
   // invalid string.
   buckets_str = "0.0,400,1.5,600";
   EXPECT_FALSE(
       ScoredHistoryMatch::GetHQPBucketsFromString(buckets_str, &hqp_buckets));
 }