Revert 212459 "Remove TextDatabase from the history service."

trunk/src/chrome/browser/history/expire_history_backend_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <algorithm>
 #include <string>
 #include <utility>
 
 #include "base/basictypes.h"
 #include "base/compiler_specific.h"
 #include "base/file_util.h"
 #include "base/files/file_path.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/path_service.h"
 #include "base/stl_util.h"
 #include "base/strings/string16.h"
 #include "base/strings/utf_string_conversions.h"
 #include "chrome/browser/bookmarks/bookmark_model.h"
 #include "chrome/browser/bookmarks/bookmark_utils.h"
 #include "chrome/browser/chrome_notification_types.h"
 #include "chrome/browser/history/archived_database.h"
 #include "chrome/browser/history/expire_history_backend.h"
 #include "chrome/browser/history/history_database.h"
 #include "chrome/browser/history/history_notifications.h"
+#include "chrome/browser/history/text_database_manager.h"
 #include "chrome/browser/history/thumbnail_database.h"
 #include "chrome/browser/history/top_sites.h"
 #include "chrome/common/thumbnail_score.h"
 #include "chrome/test/base/testing_profile.h"
 #include "chrome/tools/profiles/thumbnail-inl.h"
 #include "content/public/test/test_browser_thread.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "ui/gfx/codec/jpeg_codec.h"
 
@@ skipping 33 matching lines @@
   // Add visits with source information.
   void AddExampleSourceData(const GURL& url, URLID* id);
 
   // Returns true if the given favicon/thumanil has an entry in the DB.
   bool HasFavicon(chrome::FaviconID favicon_id);
   bool HasThumbnail(URLID url_id);
 
   chrome::FaviconID GetFavicon(const GURL& page_url,
                                chrome::IconType icon_type);
 
+  // Returns the number of text matches for the given URL in the example data
+  // added by AddExampleData.
+  int CountTextMatchesForURL(const GURL& url);
+
   // EXPECTs that each URL-specific history thing (basically, everything but
   // favicons) is gone.
   void EnsureURLInfoGone(const URLRow& row);
 
   // Clears the list of notifications received.
   void ClearLastNotifications() {
     STLDeleteValues(&notifications_);
   }
 
   void StarURL(const GURL& url) {
@@ skipping 13 matching lines @@
 
   base::MessageLoopForUI message_loop_;
   content::TestBrowserThread ui_thread_;
   content::TestBrowserThread db_thread_;
 
   ExpireHistoryBackend expirer_;
 
   scoped_ptr<HistoryDatabase> main_db_;
   scoped_ptr<ArchivedDatabase> archived_db_;
   scoped_ptr<ThumbnailDatabase> thumb_db_;
+  scoped_ptr<TextDatabaseManager> text_db_;
   TestingProfile profile_;
   scoped_refptr<TopSites> top_sites_;
 
   // Time at the beginning of the test, so everybody agrees what "now" is.
   const Time now_;
 
   // Notifications intended to be broadcast, we can check these values to make
   // sure that the deletor is doing the correct broadcasts. We own the details
   // pointers.
   typedef std::vector< std::pair<int, HistoryDetails*> >
@@ skipping 12 matching lines @@
     base::FilePath archived_name = path().Append(kArchivedHistoryFile);
     archived_db_.reset(new ArchivedDatabase);
     if (!archived_db_->Init(archived_name))
       archived_db_.reset();
 
     base::FilePath thumb_name = path().Append(kThumbnailFile);
     thumb_db_.reset(new ThumbnailDatabase);
     if (thumb_db_->Init(thumb_name, NULL, main_db_.get()) != sql::INIT_OK)
       thumb_db_.reset();
 
-    expirer_.SetDatabases(main_db_.get(), archived_db_.get(), thumb_db_.get());
+    text_db_.reset(new TextDatabaseManager(path(),
+                                           main_db_.get(), main_db_.get()));
+    if (!text_db_->Init(NULL))
+      text_db_.reset();
+
+    expirer_.SetDatabases(main_db_.get(), archived_db_.get(), thumb_db_.get(),
+                          text_db_.get());
     profile_.CreateTopSites();
     profile_.BlockUntilTopSitesLoaded();
     top_sites_ = profile_.GetTopSites();
   }
 
   virtual void TearDown() {
     top_sites_ = NULL;
 
     ClearLastNotifications();
 
-    expirer_.SetDatabases(NULL, NULL, NULL);
+    expirer_.SetDatabases(NULL, NULL, NULL, NULL);
 
     main_db_.reset();
     archived_db_.reset();
     thumb_db_.reset();
+    text_db_.reset();
   }
 
   // BroadcastNotificationDelegate implementation.
   virtual void BroadcastNotifications(
       int type,
       HistoryDetails* details_deleted) OVERRIDE {
     // This gets called when there are notifications to broadcast. Instead, we
     // store them so we can tell that the correct notifications were sent.
     notifications_.push_back(std::make_pair(type, details_deleted));
   }
   virtual void NotifySyncURLsDeleted(
       bool all_history,
       bool archived,
       URLRows* rows) OVERRIDE {}
 };
 
 // The example data consists of 4 visits. The middle two visits are to the
 // same URL, while the first and last are for unique ones. This allows a test
 // for the oldest or newest to include both a URL that should get totally
 // deleted (the one on the end) with one that should only get a visit deleted
 // (with the one in the middle) when it picks the proper threshold time.
 //
 // Each visit has indexed data, each URL has thumbnail. The first two URLs will
 // share the same avicon, while the last one will have a unique favicon. The
 // second visit for the middle URL is typed.
 //
 // The IDs of the added URLs, and the times of the four added visits will be
 // added to the given arrays.
 void ExpireHistoryTest::AddExampleData(URLID url_ids[3], Time visit_times[4]) {
-  if (!main_db_.get())
+  if (!main_db_.get() || !text_db_)
     return;
 
   // Four times for each visit.
   visit_times[3] = Time::Now();
   visit_times[2] = visit_times[3] - TimeDelta::FromDays(1);
   visit_times[1] = visit_times[3] - TimeDelta::FromDays(2);
   visit_times[0] = visit_times[3] - TimeDelta::FromDays(3);
 
   // Two favicons. The first two URLs will share the same one, while the last
   // one will have a unique favicon.
@@ skipping 31 matching lines @@
   Time time;
   GURL gurl;
   top_sites_->SetPageThumbnail(url_row1.url(), thumbnail, score);
   top_sites_->SetPageThumbnail(url_row2.url(), thumbnail, score);
   top_sites_->SetPageThumbnail(url_row3.url(), thumbnail, score);
 
   // Four visits.
   VisitRow visit_row1;
   visit_row1.url_id = url_ids[0];
   visit_row1.visit_time = visit_times[0];
+  visit_row1.is_indexed = true;
   main_db_->AddVisit(&visit_row1, SOURCE_BROWSED);
 
   VisitRow visit_row2;
   visit_row2.url_id = url_ids[1];
   visit_row2.visit_time = visit_times[1];
+  visit_row2.is_indexed = true;
   main_db_->AddVisit(&visit_row2, SOURCE_BROWSED);
 
   VisitRow visit_row3;
   visit_row3.url_id = url_ids[1];
   visit_row3.visit_time = visit_times[2];
+  visit_row3.is_indexed = true;
   visit_row3.transition = content::PAGE_TRANSITION_TYPED;
   main_db_->AddVisit(&visit_row3, SOURCE_BROWSED);
 
   VisitRow visit_row4;
   visit_row4.url_id = url_ids[2];
   visit_row4.visit_time = visit_times[3];
+  visit_row4.is_indexed = true;
   main_db_->AddVisit(&visit_row4, SOURCE_BROWSED);
+
+  // Full text index for each visit.
+  text_db_->AddPageData(url_row1.url(), visit_row1.url_id, visit_row1.visit_id,
+                        visit_row1.visit_time, UTF8ToUTF16("title"),
+                        UTF8ToUTF16("body"));
+
+  text_db_->AddPageData(url_row2.url(), visit_row2.url_id, visit_row2.visit_id,
+                        visit_row2.visit_time, UTF8ToUTF16("title"),
+                        UTF8ToUTF16("body"));
+  text_db_->AddPageData(url_row2.url(), visit_row3.url_id, visit_row3.visit_id,
+                        visit_row3.visit_time, UTF8ToUTF16("title"),
+                        UTF8ToUTF16("body"));
+
+  // Note the special text in this URL. We'll search the file for this string
+  // to make sure it doesn't hang around after the delete.
+  text_db_->AddPageData(url_row3.url(), visit_row4.url_id, visit_row4.visit_id,
+                        visit_row4.visit_time, UTF8ToUTF16("title"),
+                        UTF8ToUTF16("goats body"));
 }
 
 void ExpireHistoryTest::AddExampleSourceData(const GURL& url, URLID* id) {
   if (!main_db_)
     return;
 
   Time last_visit_time = Time::Now();
   // Add one URL.
   URLRow url_row1(url);
   url_row1.set_last_visit(last_visit_time);
@@ skipping 39 matching lines @@
   // TODO(sky): fix this. This test isn't really valid for TopSites. For
   // TopSites we should be checking URL always, not the id.
   URLRow info;
   if (!main_db_->GetURLRow(url_id, &info))
     return false;
   GURL url = info.url();
   scoped_refptr<base::RefCountedMemory> data;
   return top_sites_->GetPageThumbnail(url, &data);
 }
 
+int ExpireHistoryTest::CountTextMatchesForURL(const GURL& url) {
+  if (!text_db_)
+    return 0;
+
+  // "body" should match all pages in the example data.
+  std::vector<TextDatabase::Match> results;
+  QueryOptions options;
+  Time first_time;
+  text_db_->GetTextMatches(UTF8ToUTF16("body"), options,
+                           &results, &first_time);
+
+  int count = 0;
+  for (size_t i = 0; i < results.size(); i++) {
+    if (results[i].url == url)
+      count++;
+  }
+  return count;
+}
+
 void ExpireHistoryTest::EnsureURLInfoGone(const URLRow& row) {
   // Verify the URL no longer exists.
   URLRow temp_row;
   EXPECT_FALSE(main_db_->GetURLRow(row.id(), &temp_row));
 
+  // The indexed data should be gone.
+  EXPECT_EQ(0, CountTextMatchesForURL(row.url()));
+
   // There should be no visits.
   VisitVector visits;
   main_db_->GetVisitsForURL(row.id(), &visits);
   EXPECT_EQ(0U, visits.size());
 
   // Thumbnail should be gone.
   // TODO(sky): fix this, see comment in HasThumbnail.
   // EXPECT_FALSE(HasThumbnail(row.id()));
 
   bool found_delete_notification = false;
@@ skipping 75 matching lines @@
   URLRow last_row;
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[2], &last_row));
   chrome::FaviconID favicon_id = GetFavicon(last_row.url(), chrome::FAVICON);
   EXPECT_TRUE(HasFavicon(favicon_id));
   // TODO(sky): fix this, see comment in HasThumbnail.
   // EXPECT_TRUE(HasThumbnail(url_ids[2]));
 
   VisitVector visits;
   main_db_->GetVisitsForURL(url_ids[2], &visits);
   ASSERT_EQ(1U, visits.size());
+  EXPECT_EQ(1, CountTextMatchesForURL(last_row.url()));
+
+  // In this test we also make sure that any pending entries in the text
+  // database manager are removed.
+  text_db_->AddPageURL(last_row.url(), last_row.id(), visits[0].visit_id,
+                       visits[0].visit_time);
+
+  // Compute the text DB filename.
+  base::FilePath fts_filename = path().Append(
+      TextDatabase::IDToFileName(text_db_->TimeToID(visit_times[3])));
+
+  // When checking the file, the database must be closed. We then re-initialize
+  // it just like the test set-up did.
+  text_db_.reset();
+  EXPECT_TRUE(IsStringInFile(fts_filename, "goats"));
+  text_db_.reset(new TextDatabaseManager(path(),
+                                         main_db_.get(), main_db_.get()));
+  ASSERT_TRUE(text_db_->Init(NULL));
+  expirer_.SetDatabases(main_db_.get(), archived_db_.get(), thumb_db_.get(),
+                        text_db_.get());
 
   // Delete the URL and its dependencies.
   expirer_.DeleteURL(last_row.url());
 
+  // The string should be removed from the file. FTS can mark it as gone but
+  // doesn't remove it from the file, we want to be sure we're doing the latter.
+  text_db_.reset();
+  EXPECT_FALSE(IsStringInFile(fts_filename, "goats"));
+  text_db_.reset(new TextDatabaseManager(path(),
+                                         main_db_.get(), main_db_.get()));
+  ASSERT_TRUE(text_db_->Init(NULL));
+  expirer_.SetDatabases(main_db_.get(), archived_db_.get(), thumb_db_.get(),
+                        text_db_.get());
+
+  // Run the text database expirer. This will flush any pending entries so we
+  // can check that nothing was committed. We use a time far in the future so
+  // that anything added recently will get flushed.
+  TimeTicks expiration_time = TimeTicks::Now() + TimeDelta::FromDays(1);
+  text_db_->FlushOldChangesForTime(expiration_time);
+
   // All the normal data + the favicon should be gone.
   EnsureURLInfoGone(last_row);
   EXPECT_FALSE(GetFavicon(last_row.url(), chrome::FAVICON));
   EXPECT_FALSE(HasFavicon(favicon_id));
 }
 
 // Deletes a URL with a favicon that other URLs reference, so that the favicon
 // should not get deleted. This also tests deleting more than one visit.
 TEST_F(ExpireHistoryTest, DeleteURLWithoutFavicon) {
   URLID url_ids[3];
   Time visit_times[4];
   AddExampleData(url_ids, visit_times);
 
   // Verify things are the way we expect with a URL row, favicon, thumbnail.
   URLRow last_row;
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[1], &last_row));
   chrome::FaviconID favicon_id = GetFavicon(last_row.url(), chrome::FAVICON);
   EXPECT_TRUE(HasFavicon(favicon_id));
   // TODO(sky): fix this, see comment in HasThumbnail.
   // EXPECT_TRUE(HasThumbnail(url_ids[1]));
 
   VisitVector visits;
   main_db_->GetVisitsForURL(url_ids[1], &visits);
   EXPECT_EQ(2U, visits.size());
+  EXPECT_EQ(1, CountTextMatchesForURL(last_row.url()));
 
   // Delete the URL and its dependencies.
   expirer_.DeleteURL(last_row.url());
 
   // All the normal data except the favicon should be gone.
   EnsureURLInfoGone(last_row);
   EXPECT_TRUE(HasFavicon(favicon_id));
 }
 
 // DeleteURL should not delete starred urls.
@@ skipping 12 matching lines @@
   expirer_.DeleteURL(url_row.url());
 
   // Because the url is starred, it shouldn't be deleted.
   GURL url = url_row.url();
   ASSERT_TRUE(main_db_->GetRowForURL(url, &url_row));
 
   // And the favicon should exist.
   chrome::FaviconID favicon_id = GetFavicon(url_row.url(), chrome::FAVICON);
   EXPECT_TRUE(HasFavicon(favicon_id));
 
+  // But there should be no fts.
+  ASSERT_EQ(0, CountTextMatchesForURL(url_row.url()));
+
   // And no visits.
   VisitVector visits;
   main_db_->GetVisitsForURL(url_row.id(), &visits);
   ASSERT_EQ(0U, visits.size());
 
   // Should still have the thumbnail.
   // TODO(sky): fix this, see comment in HasThumbnail.
   // ASSERT_TRUE(HasThumbnail(url_row.id()));
 
   // Unstar the URL and delete again.
@@ skipping 46 matching lines @@
 // the two visits) and one is deleted.
 TEST_F(ExpireHistoryTest, FlushRecentURLsUnstarred) {
   URLID url_ids[3];
   Time visit_times[4];
   AddExampleData(url_ids, visit_times);
 
   URLRow url_row1, url_row2;
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[1], &url_row1));
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[2], &url_row2));
 
+  // In this test we also make sure that any pending entries in the text
+  // database manager are removed.
   VisitVector visits;
   main_db_->GetVisitsForURL(url_ids[2], &visits);
   ASSERT_EQ(1U, visits.size());
+  text_db_->AddPageURL(url_row2.url(), url_row2.id(), visits[0].visit_id,
+                       visits[0].visit_time);
 
   // This should delete the last two visits.
   std::set<GURL> restrict_urls;
   expirer_.ExpireHistoryBetween(restrict_urls, visit_times[2], Time());
 
+  // Run the text database expirer. This will flush any pending entries so we
+  // can check that nothing was committed. We use a time far in the future so
+  // that anything added recently will get flushed.
+  TimeTicks expiration_time = TimeTicks::Now() + TimeDelta::FromDays(1);
+  text_db_->FlushOldChangesForTime(expiration_time);
+
   // Verify that the middle URL had its last visit deleted only.
   visits.clear();
   main_db_->GetVisitsForURL(url_ids[1], &visits);
   EXPECT_EQ(1U, visits.size());
+  EXPECT_EQ(0, CountTextMatchesForURL(url_row1.url()));
 
   // Verify that the middle URL visit time and visit counts were updated.
   URLRow temp_row;
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[1], &temp_row));
   EXPECT_TRUE(visit_times[2] == url_row1.last_visit());  // Previous value.
   EXPECT_TRUE(visit_times[1] == temp_row.last_visit());  // New value.
   EXPECT_EQ(2, url_row1.visit_count());
   EXPECT_EQ(1, temp_row.visit_count());
   EXPECT_EQ(1, url_row1.typed_count());
   EXPECT_EQ(0, temp_row.typed_count());
@@ skipping 13 matching lines @@
 // Expires all URLs with times in a given set.
 TEST_F(ExpireHistoryTest, FlushURLsForTimes) {
   URLID url_ids[3];
   Time visit_times[4];
   AddExampleData(url_ids, visit_times);
 
   URLRow url_row1, url_row2;
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[1], &url_row1));
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[2], &url_row2));
 
+  // In this test we also make sure that any pending entries in the text
+  // database manager are removed.
   VisitVector visits;
   main_db_->GetVisitsForURL(url_ids[2], &visits);
   ASSERT_EQ(1U, visits.size());
+  text_db_->AddPageURL(url_row2.url(), url_row2.id(), visits[0].visit_id,
+                       visits[0].visit_time);
 
   // This should delete the last two visits.
   std::vector<base::Time> times;
   times.push_back(visit_times[3]);
   times.push_back(visit_times[2]);
   expirer_.ExpireHistoryForTimes(times);
 
+  // Run the text database expirer. This will flush any pending entries so we
+  // can check that nothing was committed. We use a time far in the future so
+  // that anything added recently will get flushed.
+  TimeTicks expiration_time = TimeTicks::Now() + TimeDelta::FromDays(1);
+  text_db_->FlushOldChangesForTime(expiration_time);
+
   // Verify that the middle URL had its last visit deleted only.
   visits.clear();
   main_db_->GetVisitsForURL(url_ids[1], &visits);
   EXPECT_EQ(1U, visits.size());
+  EXPECT_EQ(0, CountTextMatchesForURL(url_row1.url()));
 
   // Verify that the middle URL visit time and visit counts were updated.
   URLRow temp_row;
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[1], &temp_row));
   EXPECT_TRUE(visit_times[2] == url_row1.last_visit());  // Previous value.
   EXPECT_TRUE(visit_times[1] == temp_row.last_visit());  // New value.
   EXPECT_EQ(2, url_row1.visit_count());
   EXPECT_EQ(1, temp_row.visit_count());
   EXPECT_EQ(1, url_row1.typed_count());
   EXPECT_EQ(0, temp_row.typed_count());
@@ skipping 15 matching lines @@
 // one of the two visits).
 TEST_F(ExpireHistoryTest, FlushRecentURLsUnstarredRestricted) {
   URLID url_ids[3];
   Time visit_times[4];
   AddExampleData(url_ids, visit_times);
 
   URLRow url_row1, url_row2;
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[1], &url_row1));
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[2], &url_row2));
 
+  // In this test we also make sure that any pending entries in the text
+  // database manager are removed.
   VisitVector visits;
   main_db_->GetVisitsForURL(url_ids[2], &visits);
   ASSERT_EQ(1U, visits.size());
+  text_db_->AddPageURL(url_row2.url(), url_row2.id(), visits[0].visit_id,
+                       visits[0].visit_time);
 
   // This should delete the last two visits.
   std::set<GURL> restrict_urls;
   restrict_urls.insert(url_row1.url());
   expirer_.ExpireHistoryBetween(restrict_urls, visit_times[2], Time());
 
+  // Run the text database expirer. This will flush any pending entries so we
+  // can check that nothing was committed. We use a time far in the future so
+  // that anything added recently will get flushed.
+  TimeTicks expiration_time = TimeTicks::Now() + TimeDelta::FromDays(1);
+  text_db_->FlushOldChangesForTime(expiration_time);
+
   // Verify that the middle URL had its last visit deleted only.
   visits.clear();
   main_db_->GetVisitsForURL(url_ids[1], &visits);
   EXPECT_EQ(1U, visits.size());
+  EXPECT_EQ(0, CountTextMatchesForURL(url_row1.url()));
 
   // Verify that the middle URL visit time and visit counts were updated.
   URLRow temp_row;
   ASSERT_TRUE(main_db_->GetURLRow(url_ids[1], &temp_row));
   EXPECT_TRUE(visit_times[2] == url_row1.last_visit());  // Previous value.
   EXPECT_TRUE(visit_times[1] == temp_row.last_visit());  // New value.
   EXPECT_EQ(2, url_row1.visit_count());
   EXPECT_EQ(1, temp_row.visit_count());
   EXPECT_EQ(1, url_row1.typed_count());
   EXPECT_EQ(0, temp_row.typed_count());
@@ skipping 236 matching lines @@
   main_db_->GetVisitsForURL(url_id, &archived_visits);
   EXPECT_EQ(0U, archived_visits.size());
 }
 
 // TODO(brettw) add some visits with no URL to make sure everything is updated
 // properly. Have the visits also refer to nonexistent FTS rows.
 //
 // Maybe also refer to invalid favicons.
 
 }  // namespace history