Remove TextDatabase from the history service.

chrome/browser/history/history_backend.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/history/history_backend.h"
 
 #include <algorithm>
 #include <functional>
 #include <list>
 #include <map>
 #include <set>
 #include <vector>
 
 #include "base/basictypes.h"
 #include "base/bind.h"
 #include "base/compiler_specific.h"
 #include "base/file_util.h"
+#include "base/files/file_enumerator.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/memory/scoped_vector.h"
 #include "base/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/rand_util.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/time.h"
 #include "chrome/browser/autocomplete/history_url_provider.h"
 #include "chrome/browser/bookmarks/bookmark_service.h"
@@ skipping 34 matching lines @@
       VisitDatabase (stores a list of visits for the URLs)
       VisitSegmentDatabase (stores groups of URLs for the most visited view).
 
     ArchivedDatabase (stores history older than 3 months)
       URLDatabase (stores a list of URLs)
       DownloadDatabase (stores a list of downloads)
       VisitDatabase (stores a list of visits for the URLs)
 
       (this does not store visit segments as they expire after 3 mos.)
 
-    TextDatabaseManager (manages multiple text database for different times)
-      TextDatabase (represents a single month of full-text index).
-      ...more TextDatabase objects...
-
     ExpireHistoryBackend (manages moving things from HistoryDatabase to
                           the ArchivedDatabase and deleting)
 */
 
 namespace history {
 
 // How long we keep segment data for in days. Currently 3 months.
 // This value needs to be greater or equal to
 // MostVisitedModel::kMostVisitedScope but we don't want to introduce a direct
 // dependency between MostVisitedModel and the history backend.
@@ skipping 71 matching lines @@
   }
 
  private:
   friend class base::RefCounted<CommitLaterTask>;
 
   ~CommitLaterTask() {}
 
   scoped_refptr<HistoryBackend> history_backend_;
 };
 
-// Handles querying first the main database, then the full text database if that
-// fails. It will optionally keep track of all URLs seen so duplicates can be
-// eliminated. This is used by the querying sub-functions.
-//
-// TODO(brettw): This class may be able to be simplified or eliminated. After
-// this was written, QueryResults can efficiently look up by URL, so the need
-// for this extra set of previously queried URLs is less important.
-class HistoryBackend::URLQuerier {
- public:
-  URLQuerier(URLDatabase* main_db, URLDatabase* archived_db, bool track_unique)
-      : main_db_(main_db),
-        archived_db_(archived_db),
-        track_unique_(track_unique) {
-  }
-
-  // When we're tracking unique URLs, returns true if this URL has been
-  // previously queried. Only call when tracking unique URLs.
-  bool HasURL(const GURL& url) {
-    DCHECK(track_unique_);
-    return unique_urls_.find(url) != unique_urls_.end();
-  }
-
-  bool GetRowForURL(const GURL& url, URLRow* row) {
-    if (!main_db_->GetRowForURL(url, row)) {
-      if (!archived_db_ || !archived_db_->GetRowForURL(url, row)) {
-        // This row is neither in the main nor the archived DB.
-        return false;
-      }
-    }
-
-    if (track_unique_)
-      unique_urls_.insert(url);
-    return true;
-  }
-
- private:
-  URLDatabase* main_db_;  // Guaranteed non-NULL.
-  URLDatabase* archived_db_;  // Possibly NULL.
-
-  bool track_unique_;
-
-  // When track_unique_ is set, this is updated with every URL seen so far.
-  std::set<GURL> unique_urls_;
-
-  DISALLOW_COPY_AND_ASSIGN(URLQuerier);
-};
-
 // KillHistoryDatabaseErrorDelegate -------------------------------------------
 
 class KillHistoryDatabaseErrorDelegate : public sql::ErrorDelegate {
  public:
   explicit KillHistoryDatabaseErrorDelegate(HistoryBackend* backend)
       : backend_(backend),
         scheduled_killing_database_(false) {
   }
 
   // sql::ErrorDelegate implementation.
@@ skipping 386 matching lines @@
         // Update the visit_details for this visit.
         UpdateVisitDuration(from_visit_id, request.time);
       }
 
       // Subsequent transitions in the redirect list must all be server
       // redirects.
       redirect_info = content::PAGE_TRANSITION_SERVER_REDIRECT;
     }
 
     // Last, save this redirect chain for later so we can set titles & favicons
-    // on the redirected pages properly. It is indexed by the destination page.
+    // on the redirected pages properly.
     recent_redirects_.Put(request.url, redirects);
   }
 
   // TODO(brettw) bug 1140015: Add an "add page" notification so the history
   // views can keep in sync.
 
   // Add the last visit to the tracker so we can get outgoing transitions.
   // TODO(evanm): Due to http://b/1194536 we lose the referrers of a subframe
   // navigation anyway, so last_visit_id is always zero for them.  But adding
   // them here confuses main frame history, so we skip them for now.
   if (stripped_transition != content::PAGE_TRANSITION_AUTO_SUBFRAME &&
       stripped_transition != content::PAGE_TRANSITION_MANUAL_SUBFRAME &&
       !is_keyword_generated) {
     tracker_.AddVisit(request.id_scope, request.page_id, request.url,
                       last_ids.second);
   }
 
-  if (text_database_) {
-    text_database_->AddPageURL(request.url, last_ids.first, last_ids.second,
-                               request.time);
-  }
-
   ScheduleCommit();
 }
 
 void HistoryBackend::InitImpl(const std::string& languages) {
   DCHECK(!db_) << "Initializing HistoryBackend twice";
   // In the rare case where the db fails to initialize a dialog may get shown
   // the blocks the caller, yet allows other messages through. For this reason
   // we only set db_ to the created database if creation is successful. That
   // way other methods won't do anything as db_ is still NULL.
 
   TimeTicks beginning_time = TimeTicks::Now();
 
-  // Compute the file names. Note that the index file can be removed when the
-  // text db manager is finished being hooked up.
+  // Compute the file names.
   base::FilePath history_name = history_dir_.Append(chrome::kHistoryFilename);
   base::FilePath thumbnail_name = GetThumbnailFileName();
   base::FilePath archived_name = GetArchivedFileName();
 
+  // Delete the old index database files which are no longer used.
+  DeleteFTSIndexDatabases();
+
   // History database.
   db_.reset(new HistoryDatabase());
 
   // |HistoryDatabase::Init| takes ownership of |error_delegate|.
   KillHistoryDatabaseErrorDelegate* error_delegate =
       new KillHistoryDatabaseErrorDelegate(this);
 
   sql::InitStatus status = db_->Init(history_name, error_delegate);
   switch (status) {
     case sql::INIT_OK:
@@ skipping 19 matching lines @@
   // Fill the in-memory database and send it back to the history service on the
   // main thread.
   InMemoryHistoryBackend* mem_backend = new InMemoryHistoryBackend;
   if (mem_backend->Init(history_name, db_.get()))
     delegate_->SetInMemoryBackend(id_, mem_backend);  // Takes ownership of
                                                       // pointer.
   else
     delete mem_backend;  // Error case, run without the in-memory DB.
   db_->BeginExclusiveMode();  // Must be after the mem backend read the data.
 
-  // Create the history publisher which needs to be passed on to the text and
-  // thumbnail databases for publishing history.
+  // Create the history publisher which needs to be passed on to the thumbnail
+  // database for publishing history.
   history_publisher_.reset(new HistoryPublisher());
   if (!history_publisher_->Init()) {
     // The init may fail when there are no indexers wanting our history.
     // Hence no need to log the failure.
     history_publisher_.reset();
   }
 
-  // Full-text database. This has to be first so we can pass it to the
-  // HistoryDatabase for migration.
-  text_database_.reset(new TextDatabaseManager(history_dir_,
-                                               db_.get(), db_.get()));
-  if (!text_database_->Init(history_publisher_.get())) {
-    LOG(WARNING) << "Text database initialization failed, running without it.";
-    text_database_.reset();
-  }
-  if (db_->needs_version_17_migration()) {
-    // See needs_version_17_migration() decl for more. In this case, we want
-    // to erase all the text database files. This must be done after the text
-    // database manager has been initialized, since it knows about all the
-    // files it manages.
-    text_database_->DeleteAll();
-  }
-
   // Thumbnail database.
   thumbnail_db_.reset(new ThumbnailDatabase());
   if (!db_->GetNeedsThumbnailMigration()) {
     // No convertion needed - use new filename right away.
     thumbnail_name = GetFaviconsFileName();
   }
   if (thumbnail_db_->Init(thumbnail_name,
                           history_publisher_.get(),
                           db_.get()) != sql::INIT_OK) {
     // Unlike the main database, we don't error out when the database is too
@@ skipping 32 matching lines @@
   }
 
   // Tell the expiration module about all the nice databases we made. This must
   // happen before db_->Init() is called since the callback ForceArchiveHistory
   // may need to expire stuff.
   //
   // *sigh*, this can all be cleaned up when that migration code is removed.
   // The main DB initialization should intuitively be first (not that it
   // actually matters) and the expirer should be set last.
   expirer_.SetDatabases(db_.get(), archived_db_.get(),
-                        thumbnail_db_.get(), text_database_.get());
+                        thumbnail_db_.get());
 
   // Open the long-running transaction.
   db_->BeginTransaction();
   if (thumbnail_db_)
     thumbnail_db_->BeginTransaction();
   if (archived_db_)
     archived_db_->BeginTransaction();
-  if (text_database_)
-    text_database_->BeginTransaction();
 
   // Get the first item in our database.
   db_->GetStartDate(&first_recorded_time_);
 
   // Start expiring old stuff.
   expirer_.StartArchivingOldStuff(TimeDelta::FromDays(kArchiveDaysThreshold));
 
 #if defined(OS_ANDROID)
   if (thumbnail_db_) {
     android_provider_backend_.reset(new AndroidProviderBackend(
@@ skipping 13 matching lines @@
     db_.reset();
   }
   if (thumbnail_db_) {
     thumbnail_db_->CommitTransaction();
     thumbnail_db_.reset();
   }
   if (archived_db_) {
     archived_db_->CommitTransaction();
     archived_db_.reset();
   }
-  if (text_database_) {
-    text_database_->CommitTransaction();
-    text_database_.reset();
-  }
 }
 
 std::pair<URLID, VisitID> HistoryBackend::AddPageVisit(
     const GURL& url,
     Time time,
     VisitID referring_visit,
     content::PageTransition transition,
     VisitSource visit_source) {
   // Top-level frame navigations are visible, everything else is hidden
   bool new_hidden = !content::PageTransitionIsMainFrame(transition);
@@ skipping 34 matching lines @@
     url_info.set_typed_count(typed_increment);
     url_info.set_last_visit(time);
     url_info.set_hidden(new_hidden);
 
     url_id = db_->AddURL(url_info);
     if (!url_id) {
       NOTREACHED() << "Adding URL failed.";
       return std::make_pair(0, 0);
     }
     url_info.id_ = url_id;
-
-    // We don't actually add the URL to the full text index at this point. It
-    // might be nice to do this so that even if we get no title or body, the
-    // user can search for URL components and get the page.
-    //
-    // However, in most cases, we'll get at least a title and usually contents,
-    // and this add will be redundant, slowing everything down. As a result,
-    // we ignore this edge case.
   }
 
   // Add the visit with the time to the database.
   VisitRow visit_info(url_id, time, referring_visit, transition, 0);
   VisitID visit_id = db_->AddVisit(&visit_info, visit_source);
   NotifyVisitObservers(visit_info);
 
   if (visit_info.visit_time < first_recorded_time_)
     first_recorded_time_ = visit_info.visit_time;
 
@@ skipping 49 matching lines @@
         NOTREACHED() << "Could not add row to DB";
         return;
       }
 
       if (i->typed_count() > 0) {
         modified->changed_urls.push_back(*i);
         modified->changed_urls.back().set_id(url_id);  // *i likely has |id_| 0.
       }
     }
 
-    // Add the page to the full text index. This function is also used for
-    // importing. Even though we don't have page contents, we can at least
-    // add the title and URL to the index so they can be searched. We don't
-    // bother to delete any already-existing FTS entries for the URL, since
-    // this is normally called on import.
-    //
-    // If you ever import *after* first run (selecting import from the menu),
-    // then these additional entries will "shadow" the originals when querying
-    // for the most recent match only, and the user won't get snippets. This is
-    // a very minor issue, and fixing it will make import slower, so we don't
-    // bother.
-    bool has_indexed = false;
-    if (text_database_) {
-      // We do not have to make it update the visit database, below, we will
-      // create the visit entry with the indexed flag set.
-      has_indexed = text_database_->AddPageData(i->url(), url_id, 0,
-                                                i->last_visit(),
-                                                i->title(), string16());
-    }
-
     // Sync code manages the visits itself.
     if (visit_source != SOURCE_SYNCED) {
       // Make up a visit to correspond to the last visit to the page.
       VisitRow visit_info(url_id, i->last_visit(), 0,
                           content::PageTransitionFromInt(
                               content::PAGE_TRANSITION_LINK |
                               content::PAGE_TRANSITION_CHAIN_START |
                               content::PAGE_TRANSITION_CHAIN_END), 0);
-      visit_info.is_indexed = has_indexed;
       if (!visit_database->AddVisit(&visit_info, visit_source)) {
         NOTREACHED() << "Adding visit failed.";
         return;
       }
       NotifyVisitObservers(visit_info);
 
       if (visit_info.visit_time < first_recorded_time_)
         first_recorded_time_ = visit_info.visit_time;
     }
   }
@@ skipping 14 matching lines @@
 
 bool HistoryBackend::IsExpiredVisitTime(const base::Time& time) {
   return time < expirer_.GetCurrentArchiveTime();
 }
 
 void HistoryBackend::SetPageTitle(const GURL& url,
                                   const string16& title) {
   if (!db_)
     return;
 
-  // Update the full text index.
-  if (text_database_)
-    text_database_->AddPageTitle(url, title);
-
   // Search for recent redirects which should get the same title. We make a
   // dummy list containing the exact URL visited if there are no redirects so
   // the processing below can be the same.
   history::RedirectList dummy_list;
   history::RedirectList* redirects;
   RedirectCache::iterator iter = recent_redirects_.Get(url);
   if (iter != recent_redirects_.end()) {
     redirects = &iter->second;
 
     // This redirect chain should have the destination URL as the last item.
@@ skipping 480 matching lines @@
   for (std::vector<URLResult>::iterator it = matching_visits.begin();
        it != matching_visits.end() && result->size() < max_results; ++it) {
     result->AppendURLBySwapping(&(*it));
   }
 
   if (matching_visits.size() == result->size() &&
       options.begin_time <= first_recorded_time_)
     result->set_reached_beginning(true);
 }
 
-void HistoryBackend::QueryHistoryFTS(const string16& text_query,
-                                     const QueryOptions& options,
-                                     QueryResults* result) {
-  if (!text_database_)
-    return;
-
-  // Full text query, first get all the FTS results in the time range.
-  std::vector<TextDatabase::Match> fts_matches;
-  Time first_time_searched;
-  text_database_->GetTextMatches(text_query, options,
-                                 &fts_matches, &first_time_searched);
-
-  URLQuerier querier(db_.get(), archived_db_.get(), true);
-
-  // Now get the row and visit information for each one.
-  URLResult url_result;  // Declare outside loop to prevent re-construction.
-  for (size_t i = 0; i < fts_matches.size(); i++) {
-    if (options.max_count != 0 &&
-        static_cast<int>(result->size()) >= options.max_count)
-      break;  // Got too many items.
-
-    // Get the URL, querying the main and archived databases as necessary. If
-    // this is not found, the history and full text search databases are out
-    // of sync and we give up with this result.
-    if (!querier.GetRowForURL(fts_matches[i].url, &url_result))
-      continue;
-
-    if (!url_result.url().is_valid())
-      continue;  // Don't report invalid URLs in case of corruption.
-
-    // Copy over the FTS stuff that the URLDatabase doesn't know about.
-    // We do this with swap() to avoid copying, since we know we don't
-    // need the original any more. Note that we override the title with the
-    // one from FTS, since that will match the title_match_positions (the
-    // FTS title and the history DB title may differ).
-    url_result.set_title(fts_matches[i].title);
-    url_result.title_match_positions_.swap(
-        fts_matches[i].title_match_positions);
-    url_result.snippet_.Swap(&fts_matches[i].snippet);
-
-    // The visit time also comes from the full text search database. Since it
-    // has the time, we can avoid an extra query of the visits table.
-    url_result.set_visit_time(fts_matches[i].time);
-
-    // Add it to the vector, this will clear our |url_row| object as a
-    // result of the swap.
-    result->AppendURLBySwapping(&url_result);
-  }
-
-  if (first_time_searched <= first_recorded_time_)
-    result->set_reached_beginning(true);
-}
-
 // Frontend to GetMostRecentRedirectsFrom from the history thread.
 void HistoryBackend::QueryRedirectsFrom(
     scoped_refptr<QueryRedirectsRequest> request,
     const GURL& url) {
   if (request->canceled())
     return;
   bool success = GetMostRecentRedirectsFrom(url, &request->value);
   request->ForwardResult(request->handle(), url, success, &request->value);
 }
 
@@ skipping 239 matching lines @@
   GetRedirectsToSpecificVisit(cur_visit, redirects);
   return true;
 }
 
 void HistoryBackend::ScheduleAutocomplete(HistoryURLProvider* provider,
                                           HistoryURLProviderParams* params) {
   // ExecuteWithDB should handle the NULL database case.
   provider->ExecuteWithDB(this, db_.get(), params);
 }
 
-void HistoryBackend::SetPageContents(const GURL& url,
-                                     const string16& contents) {
-  // This is histogrammed in the text database manager.
-  if (!text_database_)
-    return;
-  text_database_->AddPageContents(url, contents);
-}
-
 void HistoryBackend::SetPageThumbnail(
     const GURL& url,
     const gfx::Image* thumbnail,
     const ThumbnailScore& score) {
   if (!db_ || !thumbnail_db_)
     return;
 
   URLRow url_row;
   URLID url_id = db_->GetRowForURL(url, &url_row);
   if (url_id) {
@@ skipping 63 matching lines @@
   if (db_) {
     // If there is no thumbnail DB, we can still record a successful migration.
     if (thumbnail_db_) {
       thumbnail_db_->RenameAndDropThumbnails(GetThumbnailFileName(),
                                              GetFaviconsFileName());
     }
     db_->ThumbnailMigrationDone();
   }
 }
 
+void HistoryBackend::DeleteFTSIndexDatabases() {
+  // Find files on disk matching the text databases file pattern so we can
+  // quickly test for and delete them.
+  base::FilePath::StringType filepattern =
+      FILE_PATH_LITERAL("History Index *");
+  base::FileEnumerator enumerator(
+      history_dir_, false, base::FileEnumerator::FILES, filepattern);
+  base::FilePath current_file;
+  while (!(current_file = enumerator.Next()).empty()) {
+    file_util::Delete(current_file, false);

[Scott Hess - ex-Googler, 2013/06/14 19:39:56]

This will potentially leave junk files.  Please use sql::Connection::Delete(),
which doesn't exist until someone reviews
https://codereview.chromium.org/17058004 .

[rmcilroy, 2013/06/17 14:11:49]

Done.

+  }

[Scott Hess - ex-Googler, 2013/06/14 19:39:56]

I think it would be reasonable to have this histogram either the number of
databases deleted, or the number of users for which any databases were deleted.

[rmcilroy, 2013/06/17 14:11:49]

Done.

+}
+
 bool HistoryBackend::GetThumbnailFromOlderRedirect(
     const GURL& page_url,
     std::vector<unsigned char>* data) {
   // Look at a few previous visit sessions.
   VisitVector older_sessions;
   URLID page_url_id = db_->GetRowForURL(page_url, NULL);
   static const int kVisitsToSearchForThumbnail = 4;
   db_->GetMostRecentVisitsForURL(
       page_url_id, kVisitsToSearchForThumbnail, &older_sessions);
 
@@ skipping 749 matching lines @@
     thumbnail_db_->CommitTransaction();
     DCHECK(thumbnail_db_->transaction_nesting() == 0) <<
         "Somebody left a transaction open";
     thumbnail_db_->BeginTransaction();
   }
 
   if (archived_db_) {
     archived_db_->CommitTransaction();
     archived_db_->BeginTransaction();
   }
-
-  if (text_database_) {
-    text_database_->CommitTransaction();
-    text_database_->BeginTransaction();
-  }
 }
 
 void HistoryBackend::ScheduleCommit() {
   if (scheduled_commit_.get())
     return;
   scheduled_commit_ = new CommitLaterTask(this);
   base::MessageLoop::current()->PostDelayedTask(
       FROM_HERE,
       base::Bind(&CommitLaterTask::RunCommit, scheduled_commit_.get()),
       base::TimeDelta::FromSeconds(kCommitIntervalSeconds));
@@ skipping 196 matching lines @@
   bool success = db_->Raze();
   UMA_HISTOGRAM_BOOLEAN("History.KillHistoryDatabaseResult", success);
 
 #if defined(OS_ANDROID)
   // Release AndroidProviderBackend before other objects.
   android_provider_backend_.reset();
 #endif
 
   // The expirer keeps tabs on the active databases. Tell it about the
   // databases which will be closed.
-  expirer_.SetDatabases(NULL, NULL, NULL, NULL);
+  expirer_.SetDatabases(NULL, NULL, NULL);
 
   // Reopen a new transaction for |db_| for the sake of CloseAllDatabases().
   db_->BeginTransaction();
   CloseAllDatabases();
 }
 
 void HistoryBackend::ProcessDBTask(
     scoped_refptr<HistoryDBTaskRequest> request) {
   DCHECK(request.get());
   if (request->canceled())
@@ skipping 69 matching lines @@
     // We continue in this error case. If the user wants to delete their
     // history, we should delete as much as we can.
   }
 
   // ClearAllMainHistory will change the IDs of the URLs in kept_urls. Therfore,
   // we clear the list afterwards to make sure nobody uses this invalid data.
   if (!ClearAllMainHistory(kept_urls))
     LOG(ERROR) << "Main history could not be cleared";
   kept_urls.clear();
 
-  // Delete FTS files & archived history.
-  if (text_database_) {
-    // We assume that the text database has one transaction on them that we need
-    // to close & restart (the long-running history transaction).
-    text_database_->CommitTransaction();
-    text_database_->DeleteAll();
-    text_database_->BeginTransaction();
-  }
-
+  // Delete archived history.
   if (archived_db_) {
     // Close the database and delete the file.
     archived_db_.reset();
     base::FilePath archived_file_name = GetArchivedFileName();
     file_util::Delete(archived_file_name, false);
 
     // Now re-initialize the database (which may fail).
     archived_db_.reset(new ArchivedDatabase());
     if (!archived_db_->Init(archived_file_name)) {
       LOG(WARNING) << "Could not initialize the archived database.";
@@ skipping 125 matching lines @@
   info.url_id = visit.url_id;
   info.time = visit.visit_time;
   info.transition = visit.transition;
   // If we don't have a delegate yet during setup or shutdown, we will drop
   // these notifications.
   if (delegate_)
     delegate_->NotifyVisitDBObserversOnAddVisit(info);
 }
 
 }  // namespace history