Don't strip punctuation inside quotes in history search queries.

chrome/browser/history/text_database_manager_unittest.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/file_path.h"
 #include "base/file_util.h"
 #include "base/message_loop.h"
 #include "base/utf_string_conversions.h"
 #include "chrome/browser/history/text_database_manager.h"
 #include "chrome/browser/history/visit_database.h"
@@ skipping 14 matching lines @@
 
 const char* kURL2 = "http://www.google.com/qwer";
 const char* kTitle2 = "Google B";
 const char* kBody2 = "FOO two.";
 
 const char* kURL3 = "http://www.google.com/zxcv";
 const char* kTitle3 = "Google C";
 const char* kBody3 = "FOO drei";
 
 const char* kURL4 = "http://www.google.com/hjkl";
-const char* kTitle4 = "Google D";
+const char* kTitle4 = "Google de";
 const char* kBody4 = "FOO lalala four.";
 
-const char* kURL5 = "http://www.google.com/uiop";
+const char* kURL5 = "http://www.google.de/s?q=mar%10&num=66";
 const char* kTitle5 = "Google cinq";
-const char* kBody5 = "FOO page one.";
+const char* kBody5 = "FOO page one. Including: punctuation.";
 
 // This provides a simple implementation of a URL+VisitDatabase using an
 // in-memory sqlite connection. The text database manager expects to be able to
 // update the visit database to keep in sync.
 class InMemDB : public URLDatabase, public VisitDatabase {
  public:
   InMemDB() {
     EXPECT_TRUE(db_.OpenInMemory());
     CreateURLTable(false);
     InitVisitTable();
@@ skipping 25 matching lines @@
   visit_row.url_id = 1;
   visit_row.visit_time = Time::FromUTCExploded(exploded);
   visit_row.referring_visit = 0;
   visit_row.transition = content::PageTransitionFromInt(0);
   visit_row.segment_id = 0;
   visit_row.is_indexed = false;
   visit_db->AddVisit(&visit_row, SOURCE_BROWSED);
 
   times->push_back(visit_row.visit_time);
   manager.AddPageData(GURL(kURL1), visit_row.url_id, visit_row.visit_id,
-                      visit_row.visit_time, UTF8ToUTF16(kTitle1),
-                      UTF8ToUTF16(kBody1));
+                      visit_row.visit_time, ASCIIToUTF16(kTitle1),
+                      ASCIIToUTF16(kBody1));
 
   exploded.day_of_month++;
   visit_row.url_id = 2;
   visit_row.visit_time = Time::FromUTCExploded(exploded);
   visit_db->AddVisit(&visit_row, SOURCE_BROWSED);
   times->push_back(visit_row.visit_time);
   manager.AddPageData(GURL(kURL2), visit_row.url_id, visit_row.visit_id,
-                      visit_row.visit_time, UTF8ToUTF16(kTitle2),
-                      UTF8ToUTF16(kBody2));
+                      visit_row.visit_time, ASCIIToUTF16(kTitle2),
+                      ASCIIToUTF16(kBody2));
 
   exploded.day_of_month++;
   visit_row.url_id = 2;
   visit_row.visit_time = Time::FromUTCExploded(exploded);
   visit_db->AddVisit(&visit_row, SOURCE_BROWSED);
   times->push_back(visit_row.visit_time);
   manager.AddPageData(GURL(kURL3), visit_row.url_id, visit_row.visit_id,
-                      visit_row.visit_time, UTF8ToUTF16(kTitle3),
-                      UTF8ToUTF16(kBody3));
+                      visit_row.visit_time, ASCIIToUTF16(kTitle3),
+                      ASCIIToUTF16(kBody3));
 
   // Put the next ones in the next month.
   exploded.month++;
   visit_row.url_id = 2;
   visit_row.visit_time = Time::FromUTCExploded(exploded);
   visit_db->AddVisit(&visit_row, SOURCE_BROWSED);
   times->push_back(visit_row.visit_time);
   manager.AddPageData(GURL(kURL4), visit_row.url_id, visit_row.visit_id,
-                      visit_row.visit_time, UTF8ToUTF16(kTitle4),
-                      UTF8ToUTF16(kBody4));
+                      visit_row.visit_time, ASCIIToUTF16(kTitle4),
+                      ASCIIToUTF16(kBody4));
 
   exploded.day_of_month++;
   visit_row.url_id = 2;
   visit_row.visit_time = Time::FromUTCExploded(exploded);
   visit_db->AddVisit(&visit_row, SOURCE_BROWSED);
   times->push_back(visit_row.visit_time);
   manager.AddPageData(GURL(kURL5), visit_row.url_id, visit_row.visit_id,
-                      visit_row.visit_time, UTF8ToUTF16(kTitle5),
-                      UTF8ToUTF16(kBody5));
+                      visit_row.visit_time, ASCIIToUTF16(kTitle5),
+                      ASCIIToUTF16(kBody5));
 
   // Put the first one in again in the second month.
   exploded.day_of_month++;
   visit_row.url_id = 2;
   visit_row.visit_time = Time::FromUTCExploded(exploded);
   visit_db->AddVisit(&visit_row, SOURCE_BROWSED);
   times->push_back(visit_row.visit_time);
   manager.AddPageData(GURL(kURL1), visit_row.url_id, visit_row.visit_id,
-                      visit_row.visit_time, UTF8ToUTF16(kTitle1),
-                      UTF8ToUTF16(kBody1));
+                      visit_row.visit_time, ASCIIToUTF16(kTitle1),
+                      ASCIIToUTF16(kBody1));
 }
 
 bool ResultsHaveURL(const std::vector<TextDatabase::Match>& results,
                     const char* url) {
   GURL gurl(url);
   for (size_t i = 0; i < results.size(); i++) {
     if (results[i].url == gurl)
       return true;
   }
   return false;
@@ skipping 31 matching lines @@
   ASSERT_TRUE(manager.Init(NULL));
 
   std::vector<Time> times;
   AddAllPages(manager, &visit_db, &times);
 
   QueryOptions options;
   options.begin_time = times[0] - TimeDelta::FromDays(100);
   options.end_time = times[times.size() - 1] + TimeDelta::FromDays(100);
   std::vector<TextDatabase::Match> results;
   Time first_time_searched;
-  manager.GetTextMatches(UTF8ToUTF16("FOO"), options,
+  manager.GetTextMatches(ASCIIToUTF16("FOO"), options,
                          &results, &first_time_searched);
 
   // We should have matched every page.
   EXPECT_EQ(6U, results.size());
   EXPECT_TRUE(ResultsHaveURL(results, kURL1));
   EXPECT_TRUE(ResultsHaveURL(results, kURL2));
   EXPECT_TRUE(ResultsHaveURL(results, kURL3));
   EXPECT_TRUE(ResultsHaveURL(results, kURL4));
   EXPECT_TRUE(ResultsHaveURL(results, kURL5));
 
   // The first time searched should have been the first page's time or before
   // (it could have eliminated some time for us).
   EXPECT_TRUE(first_time_searched <= times[0]);
 }
 
 // Tests that adding page components piecemeal will get them added properly.
 // This does not supply a visit to update, this mode is used only by the unit
 // tests right now, but we test it anyway.
 TEST_F(TextDatabaseManagerTest, InsertCompleteNoVisit) {
   ASSERT_TRUE(Init());
   InMemDB visit_db;
   TextDatabaseManager manager(dir_, &visit_db, &visit_db);
   ASSERT_TRUE(manager.Init(NULL));
 
   // First add one without a visit.
   const GURL url(kURL1);
   manager.AddPageURL(url, 0, 0, Time::Now());
-  manager.AddPageTitle(url, UTF8ToUTF16(kTitle1));
-  manager.AddPageContents(url, UTF8ToUTF16(kBody1));
+  manager.AddPageTitle(url, ASCIIToUTF16(kTitle1));
+  manager.AddPageContents(url, ASCIIToUTF16(kBody1));
 
   // Check that the page got added.
   QueryOptions options;
   std::vector<TextDatabase::Match> results;
   Time first_time_searched;
 
-  manager.GetTextMatches(UTF8ToUTF16("FOO"), options,
+  manager.GetTextMatches(ASCIIToUTF16("FOO"), options,
                          &results, &first_time_searched);
   ASSERT_EQ(1U, results.size());
   EXPECT_EQ(kTitle1, UTF16ToUTF8(results[0].title));
 }
 
 // Like InsertCompleteNoVisit but specifies a visit to update. We check that the
 // visit was updated properly.
 TEST_F(TextDatabaseManagerTest, InsertCompleteVisit) {
   ASSERT_TRUE(Init());
   InMemDB visit_db;
   TextDatabaseManager manager(dir_, &visit_db, &visit_db);
   ASSERT_TRUE(manager.Init(NULL));
 
   // First add a visit to a page. We can just make up a URL ID since there is
   // not actually any URL database around.
   VisitRow visit;
   visit.url_id = 1;
   visit.visit_time = Time::Now();
   visit.referring_visit = 0;
   visit.transition = content::PAGE_TRANSITION_LINK;
   visit.segment_id = 0;
   visit.is_indexed = false;
   visit_db.AddVisit(&visit, SOURCE_BROWSED);
 
   // Add a full text indexed entry for that visit.
   const GURL url(kURL2);
   manager.AddPageURL(url, visit.url_id, visit.visit_id, visit.visit_time);
-  manager.AddPageContents(url, UTF8ToUTF16(kBody2));
-  manager.AddPageTitle(url, UTF8ToUTF16(kTitle2));
+  manager.AddPageContents(url, ASCIIToUTF16(kBody2));
+  manager.AddPageTitle(url, ASCIIToUTF16(kTitle2));
 
   // Check that the page got added.
   QueryOptions options;
   std::vector<TextDatabase::Match> results;
   Time first_time_searched;
 
-  manager.GetTextMatches(UTF8ToUTF16("FOO"), options,
+  manager.GetTextMatches(ASCIIToUTF16("FOO"), options,
                          &results, &first_time_searched);
   ASSERT_EQ(1U, results.size());
   EXPECT_EQ(kTitle2, UTF16ToUTF8(results[0].title));
 
   // Check that the visit got updated for its new indexed state.
   VisitRow out_visit;
   ASSERT_TRUE(visit_db.GetRowForVisit(visit.visit_id, &out_visit));
   EXPECT_TRUE(out_visit.is_indexed);
 }
 
 // Tests that partial inserts that expire are added to the database.
 TEST_F(TextDatabaseManagerTest, InsertPartial) {
   ASSERT_TRUE(Init());
   InMemDB visit_db;
   TextDatabaseManager manager(dir_, &visit_db, &visit_db);
   ASSERT_TRUE(manager.Init(NULL));
 
   // Add the first one with just a URL.
   GURL url1(kURL1);
   manager.AddPageURL(url1, 0, 0, Time::Now());
 
   // Now add a second one with a URL and title.
   GURL url2(kURL2);
   manager.AddPageURL(url2, 0, 0, Time::Now());
-  manager.AddPageTitle(url2, UTF8ToUTF16(kTitle2));
+  manager.AddPageTitle(url2, ASCIIToUTF16(kTitle2));
 
   // The third one has a URL and body.
   GURL url3(kURL3);
   manager.AddPageURL(url3, 0, 0, Time::Now());
-  manager.AddPageContents(url3, UTF8ToUTF16(kBody3));
+  manager.AddPageContents(url3, ASCIIToUTF16(kBody3));
 
   // Expire stuff very fast. This assumes that the time between the first
   // AddPageURL and this line is less than the expiration time (20 seconds).
   TimeTicks added_time = TimeTicks::Now();
   TimeTicks expire_time = added_time + TimeDelta::FromSeconds(5);
   manager.FlushOldChangesForTime(expire_time);
 
   // Do a query, nothing should be added yet.
   QueryOptions options;
   std::vector<TextDatabase::Match> results;
   Time first_time_searched;
-  manager.GetTextMatches(UTF8ToUTF16("google"), options,
+  manager.GetTextMatches(ASCIIToUTF16("google"), options,
                          &results, &first_time_searched);
   ASSERT_EQ(0U, results.size());
 
   // Compute a time threshold that will cause everything to be flushed, and
   // poke at the manager's internals to cause this to happen.
   expire_time = added_time + TimeDelta::FromDays(1);
   manager.FlushOldChangesForTime(expire_time);
 
   // Now we should have all 3 URLs added.
-  manager.GetTextMatches(UTF8ToUTF16("google"), options,
+  manager.GetTextMatches(ASCIIToUTF16("google"), options,
                          &results, &first_time_searched);
   ASSERT_EQ(3U, results.size());
   EXPECT_TRUE(ResultsHaveURL(results, kURL1));
   EXPECT_TRUE(ResultsHaveURL(results, kURL2));
   EXPECT_TRUE(ResultsHaveURL(results, kURL3));
 }
 
 // Tests that partial inserts (due to timeouts) will still get updated if the
 // data comes in later.
 TEST_F(TextDatabaseManagerTest, PartialComplete) {
   ASSERT_TRUE(Init());
   InMemDB visit_db;
   TextDatabaseManager manager(dir_, &visit_db, &visit_db);
   ASSERT_TRUE(manager.Init(NULL));
 
   Time added_time = Time::Now();
   GURL url(kURL1);
 
   // We have to have the URL in the URL and visit databases for this test to
   // work.
   URLRow url_row(url);
-  url_row.set_title(UTF8ToUTF16("chocolate"));
+  url_row.set_title(ASCIIToUTF16("chocolate"));
   URLID url_id = visit_db.AddURL(url_row);
   ASSERT_TRUE(url_id);
   VisitRow visit_row;
   visit_row.url_id = url_id;
   visit_row.visit_time = added_time;
   visit_db.AddVisit(&visit_row, SOURCE_BROWSED);
 
   // Add a URL with no title or body, and say that it expired.
   manager.AddPageURL(url, 0, 0, added_time);
   TimeTicks expire_time = TimeTicks::Now() + TimeDelta::FromDays(1);
   manager.FlushOldChangesForTime(expire_time);
 
   // Add the title. We should be able to query based on that. The title in the
   // URL row we set above should not come into the picture.
-  manager.AddPageTitle(url, UTF8ToUTF16("Some unique title"));
+  manager.AddPageTitle(url, ASCIIToUTF16("Some unique title"));
   Time first_time_searched;
   QueryOptions options;
   std::vector<TextDatabase::Match> results;
-  manager.GetTextMatches(UTF8ToUTF16("unique"), options,
+  manager.GetTextMatches(ASCIIToUTF16("unique"), options,
                          &results, &first_time_searched);
   EXPECT_EQ(1U, results.size());
-  manager.GetTextMatches(UTF8ToUTF16("chocolate"), options,
+  manager.GetTextMatches(ASCIIToUTF16("chocolate"), options,
                          &results, &first_time_searched);
   EXPECT_EQ(0U, results.size());
 
   // Now add the body, which should be queryable.
-  manager.AddPageContents(url, UTF8ToUTF16("Very awesome body"));
-  manager.GetTextMatches(UTF8ToUTF16("awesome"), options, &results, &first_time_searched);
+  manager.AddPageContents(url, ASCIIToUTF16("Very awesome body"));
+  manager.GetTextMatches(
+      ASCIIToUTF16("awesome"), options, &results, &first_time_searched);
   EXPECT_EQ(1U, results.size());
 
   // Adding the body will actually copy the title from the URL table rather
   // than the previously indexed row (we made them not match above). This isn't
   // necessarily what we want, but it's how it's implemented, and we don't want
   // to regress it.
-  manager.GetTextMatches(UTF8ToUTF16("chocolate"), options, &results, &first_time_searched);
+  manager.GetTextMatches(
+      ASCIIToUTF16("chocolate"), options, &results, &first_time_searched);
   EXPECT_EQ(1U, results.size());
 }
 
 // Tests that changes get properly committed to disk.
 TEST_F(TextDatabaseManagerTest, Writing) {
   ASSERT_TRUE(Init());
 
   QueryOptions options;
   std::vector<TextDatabase::Match> results;
   Time first_time_searched;
 
   InMemDB visit_db;
 
   // Create the manager and write some stuff to it.
   {
     TextDatabaseManager manager(dir_, &visit_db, &visit_db);
     ASSERT_TRUE(manager.Init(NULL));
 
     std::vector<Time> times;
     AddAllPages(manager, &visit_db, &times);
 
     // We should have matched every page.
-    manager.GetTextMatches(UTF8ToUTF16("FOO"), options, &results, &first_time_searched);
+    manager.GetTextMatches(
+        ASCIIToUTF16("FOO"), options, &results, &first_time_searched);
     EXPECT_EQ(6U, results.size());
   }
   results.clear();
 
   // Recreate the manager and make sure it finds the written stuff.
   {
     TextDatabaseManager manager(dir_, &visit_db, &visit_db);
     ASSERT_TRUE(manager.Init(NULL));
 
     // We should have matched every page again.
-    manager.GetTextMatches(UTF8ToUTF16("FOO"), options,
+    manager.GetTextMatches(ASCIIToUTF16("FOO"), options,
                            &results, &first_time_searched);
     EXPECT_EQ(6U, results.size());
   }
 }
 
 // Tests that changes get properly committed to disk, as in the Writing test
 // above, but when there is a transaction around the adds.
 TEST_F(TextDatabaseManagerTest, WritingTransaction) {
   ASSERT_TRUE(Init());
 
   QueryOptions options;
   std::vector<TextDatabase::Match> results;
   Time first_time_searched;
 
   InMemDB visit_db;
 
   // Create the manager and write some stuff to it.
   {
     TextDatabaseManager manager(dir_, &visit_db, &visit_db);
     ASSERT_TRUE(manager.Init(NULL));
 
     std::vector<Time> times;
     manager.BeginTransaction();
     AddAllPages(manager, &visit_db, &times);
     // "Forget" to commit, it should be autocommittedd for us.
 
     // We should have matched every page.
-    manager.GetTextMatches(UTF8ToUTF16("FOO"), options,
+    manager.GetTextMatches(ASCIIToUTF16("FOO"), options,
                            &results, &first_time_searched);
     EXPECT_EQ(6U, results.size());
   }
   results.clear();
 
   // Recreate the manager and make sure it finds the written stuff.
   {
     TextDatabaseManager manager(dir_, &visit_db, &visit_db);
     ASSERT_TRUE(manager.Init(NULL));
 
     // We should have matched every page again.
-    manager.GetTextMatches(UTF8ToUTF16("FOO"), options,
+    manager.GetTextMatches(ASCIIToUTF16("FOO"), options,
                            &results, &first_time_searched);
     EXPECT_EQ(6U, results.size());
   }
 }
 
 // Tests querying where the maximum number of items is met.
 TEST_F(TextDatabaseManagerTest, QueryMax) {
   ASSERT_TRUE(Init());
   InMemDB visit_db;
   TextDatabaseManager manager(dir_, &visit_db, &visit_db);
   ASSERT_TRUE(manager.Init(NULL));
 
   std::vector<Time> times;
   AddAllPages(manager, &visit_db, &times);
 
-  string16 foo = UTF8ToUTF16("FOO");
+  string16 foo = ASCIIToUTF16("FOO");
 
   QueryOptions options;
   options.begin_time = times[0] - TimeDelta::FromDays(100);
   options.end_time = times[times.size() - 1] + TimeDelta::FromDays(100);
   options.max_count = 2;
   std::vector<TextDatabase::Match> results;
   Time first_time_searched;
   manager.GetTextMatches(foo, options, &results, &first_time_searched);
 
   // We should have gotten the last two pages as results (the first page is
@@ skipping 18 matching lines @@
 // Tests querying backwards in time in chunks.
 TEST_F(TextDatabaseManagerTest, QueryBackwards) {
   ASSERT_TRUE(Init());
   InMemDB visit_db;
   TextDatabaseManager manager(dir_, &visit_db, &visit_db);
   ASSERT_TRUE(manager.Init(NULL));
 
   std::vector<Time> times;
   AddAllPages(manager, &visit_db, &times);
 
-  string16 foo = UTF8ToUTF16("FOO");
+  string16 foo = ASCIIToUTF16("FOO");
 
   // First do a query for all time, but with a max of 2. This will give us the
   // last two results and will tell us where to start searching when we want
   // to go back in time.
   QueryOptions options;
   options.begin_time = times[0] - TimeDelta::FromDays(100);
   options.end_time = times[times.size() - 1] + TimeDelta::FromDays(100);
   options.max_count = 2;
   std::vector<TextDatabase::Match> results;
   Time first_time_searched;
@@ skipping 20 matching lines @@
   EXPECT_TRUE(first_time_searched <= times[0]);
   EXPECT_TRUE(ResultsHaveURL(results, kURL2));
   EXPECT_TRUE(ResultsHaveURL(results, kURL1));
 
   // Try to query some more, there should be no results.
   options.end_time = first_time_searched;
   manager.GetTextMatches(foo, options, &results, &first_time_searched);
   EXPECT_EQ(0U, results.size());
 }
 
+TEST_F(TextDatabaseManagerTest, Query) {
+  ASSERT_TRUE(Init());
+
+  QueryOptions options;
+  std::vector<TextDatabase::Match> results;
+  Time first_time_searched;
+
+  InMemDB visit_db;
+
+  TextDatabaseManager manager(dir_, &visit_db, &visit_db);
+  ASSERT_TRUE(manager.Init(NULL));
+
+  std::vector<Time> times;
+  AddAllPages(manager, &visit_db, &times);
+
+  // Try a multi-word query.
+  manager.GetTextMatches(ASCIIToUTF16("FOO drei"), options,
+                         &results, &first_time_searched);
+  EXPECT_EQ(1U, results.size());
+  results.clear();
+
+  // Try a quoted query that should match only a single URL.
+  manager.GetTextMatches(ASCIIToUTF16("\"google.de\""),
+                         options, &results, &first_time_searched);
+  EXPECT_EQ(1U, results.size());
+  results.clear();
+
+  // Same, but with some special characters.
+  manager.GetTextMatches(ASCIIToUTF16("\"google.de/s?q=mar%10&num=66\""),
+                         options, &results, &first_time_searched);
+  EXPECT_EQ(1U, results.size());
+  results.clear();
+
+  // Using separate tokens should match in URL or in body.
+  manager.GetTextMatches(ASCIIToUTF16("google de"), options,
+                         &results, &first_time_searched);
+  EXPECT_EQ(2U, results.size());
+  EXPECT_TRUE(ResultsHaveURL(results, kURL4));
+  EXPECT_TRUE(ResultsHaveURL(results, kURL5));
+  results.clear();
+
+  // Try a quoted query that should match the body.
+  manager.GetTextMatches(ASCIIToUTF16("\"Including: punctuation.\""), options,
+                         &results, &first_time_searched);
+  EXPECT_EQ(1U, results.size());
+  results.clear();
+
+  // Ensure that punctuation is not ignored in quoted strings.
+  manager.GetTextMatches(ASCIIToUTF16("\"Including punctuation\""), options,
+                         &results, &first_time_searched);
+  EXPECT_EQ(0U, results.size());

[Scott Hess - ex-Googler, 2012/02/10 22:22:32]

I don't understand how this one is working.  When I test against a raw fts3
table, MATCH '"Including: punctiotion."' gives the same results as MATCH
'"Including punctuation"'.  Is there post-query filtering which is different
because of this change?

[Patrick Dubroy, 2012/02/11 10:58:39]

Weird. It doesn't look like there is any post-query filtering going on.
And when I test on my history index, here's the behaviour I see:

/usr/local/google/dubroy/chromium/src$ out/Debug/sqlite_shell
~/.config/chromium/Default/History\ Index\ 2012-02
SQLite version 3.7.6.3
Enter ".help" for instructions
Enter SQL statements terminated with a ";"
sqlite> select url from pages where pages match 'punctuation';
http://www.corp.google.com/~dubroy/no_crawl/fts_test.html
sqlite> select url from pages where pages match '"Including: punctuation"';
http://www.corp.google.com/~dubroy/no_crawl/fts_test.html
sqlite> select url from pages where pages match '"Including punctuation"';
sqlite>

So it seems consistent with unit test behaviour. Am I doing something different
than you did?

[Scott Hess - ex-Googler, 2012/02/11 17:09:55]

Here's what I mean:

sqlite> CREATE VIRTUAL TABLE t USING fts3(x);
sqlite> INSERT INTO t VALUES ("FOO page one. Including: punctuation.");
sqlite> SELECT * FROM t WHERE t MATCH 'punctuation';
FOO page one. Including: punctuation.
sqlite> SELECT * FROM t WHERE t MATCH '"Including: punctuation"';
FOO page one. Including: punctuation.
sqlite> SELECT * FROM t WHERE t MATCH '"Including punctuation"';
FOO page one. Including: punctuation.
sqlite> SELECT * FROM t WHERE t MATCH '"Including_ punctuation!"';
FOO page one. Including: punctuation.

[Patrick Dubroy, 2012/02/20 11:00:36]

We figured out that this was caused by the fact that the simple tokenizer is
used by default, which strips all punctuation characters. Chrome uses the ICU
tokenizer, which does not.

+
+  // Ensure that regex characters don't have special meaning inside quotes.
+  manager.GetTextMatches(ASCIIToUTF16("\"Including* punctuation\""), options,
+                         &results, &first_time_searched);
+  EXPECT_EQ(0U, results.size());
+  manager.GetTextMatches(ASCIIToUTF16("\"Including. punctuation\""), options,
+                         &results, &first_time_searched);
+  EXPECT_EQ(0U, results.size());
+  manager.GetTextMatches(ASCIIToUTF16("\"Including_ punctuation\""), options,
+                         &results, &first_time_searched);
+  EXPECT_EQ(0U, results.size());
+}
+
 }  // namespace history