Move InMemoryURLIndex Caching Operations to FILE Thread

chrome/browser/history/url_index_private_data.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/history/url_index_private_data.h"
 
 #include <algorithm>
 #include <functional>
 #include <iterator>
 #include <limits>
 #include <numeric>
 
 #include "base/file_util.h"
 #include "base/i18n/case_conversion.h"
 #include "base/metrics/histogram.h"
 #include "base/string_util.h"
-#include "base/threading/thread_restrictions.h"
+#include "base/time.h"
 #include "base/utf_string_conversions.h"
 #include "chrome/browser/autocomplete/autocomplete.h"
 #include "chrome/browser/history/history_database.h"
-#include "chrome/common/url_constants.h"
+#include "chrome/browser/history/in_memory_url_index.h"
+#include "content/public/browser/notification_details.h"
+#include "content/public/browser/notification_service.h"
+#include "content/public/browser/notification_source.h"
 #include "net/base/net_util.h"
 #include "third_party/protobuf/src/google/protobuf/repeated_field.h"
 
 using google::protobuf::RepeatedField;
 using google::protobuf::RepeatedPtrField;
 using in_memory_url_index::InMemoryURLIndexCacheItem;
 
 namespace history {
 
 typedef imui::InMemoryURLIndexCacheItem_WordListItem WordListItem;
@@ skipping 84 matching lines @@
 }
 
 // InMemoryURLIndex's Private Data ---------------------------------------------
 
 URLIndexPrivateData::URLIndexPrivateData()
     : restored_cache_version_(0),
       saved_cache_version_(kCurrentCacheFileVersion),
       pre_filter_item_count_(0),
       post_filter_item_count_(0),
       post_scoring_item_count_(0) {
-  URLIndexPrivateData::InitializeSchemeWhitelist(&scheme_whitelist_);
 }
 
 URLIndexPrivateData::~URLIndexPrivateData() {}
 
 void URLIndexPrivateData::Clear() {
   word_list_.clear();
   available_words_.clear();
   word_map_.clear();
   char_word_map_.clear();
   word_id_history_map_.clear();
   history_id_word_map_.clear();
   history_info_map_.clear();
   word_starts_map_.clear();
 }
 
+bool URLIndexPrivateData::Empty() const {
+  return history_info_map_.empty();
+}
+
+scoped_refptr<URLIndexPrivateData> URLIndexPrivateData::Duplicate() const {
+  scoped_refptr<URLIndexPrivateData> data_copy = new URLIndexPrivateData;
+  data_copy->word_list_ = word_list_;
+  data_copy->available_words_ = available_words_;
+  data_copy->word_map_ = word_map_;
+  data_copy->char_word_map_ = char_word_map_;
+  data_copy->word_id_history_map_ = word_id_history_map_;
+  data_copy->history_id_word_map_ = history_id_word_map_;
+  data_copy->history_info_map_ = history_info_map_;
+  return data_copy;
+  // Not copied:
+  //    search_term_cache_
+  //    pre_filter_item_count_
+  //    post_filter_item_count_
+  //    post_scoring_item_count_
+};
+
 // Cache Updating --------------------------------------------------------------
 
-bool URLIndexPrivateData::IndexRow(const URLRow& row) {
+bool URLIndexPrivateData::IndexRow(
+    const URLRow& row,
+    const std::string& languages,
+    const std::set<std::string>& scheme_whitelist) {
   const GURL& gurl(row.url());
 
   // Index only URLs with a whitelisted scheme.
-  if (!URLIndexPrivateData::URLSchemeIsWhitelisted(gurl))
+  if (!URLSchemeIsWhitelisted(gurl, scheme_whitelist))
     return false;
 
   URLID row_id = row.id();
   // Strip out username and password before saving and indexing.
-  string16 url(net::FormatUrl(gurl, languages_,
+  string16 url(net::FormatUrl(gurl, languages,
       net::kFormatUrlOmitUsernamePassword,
       net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS,
       NULL, NULL, NULL));
 
   HistoryID history_id = static_cast<HistoryID>(row_id);
   DCHECK_LT(history_id, std::numeric_limits<HistoryID>::max());
 
   // Add the row for quick lookup in the history info store.
   URLRow new_row(GURL(url), row_id);
   new_row.set_visit_count(row.visit_count());
   new_row.set_typed_count(row.typed_count());
   new_row.set_last_visit(row.last_visit());
   new_row.set_title(row.title());
   history_info_map_[history_id] = new_row;
 
   // Index the words contained in the URL and title of the row.
   RowWordStarts word_starts;
-  AddRowWordsToIndex(new_row, &word_starts);
+  AddRowWordsToIndex(new_row, &word_starts, languages);
   word_starts_map_[history_id] = word_starts;
   return true;
 }
 
 void URLIndexPrivateData::AddRowWordsToIndex(const URLRow& row,
-                                             RowWordStarts* word_starts) {
+                                             RowWordStarts* word_starts,
+                                             const std::string& languages) {
   HistoryID history_id = static_cast<HistoryID>(row.id());
   // Split URL into individual, unique words then add in the title words.
   const GURL& gurl(row.url());
-  string16 url(net::FormatUrl(gurl, languages_,
+  string16 url(net::FormatUrl(gurl, languages,
       net::kFormatUrlOmitUsernamePassword,
       net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS,
       NULL, NULL, NULL));
   url = base::i18n::ToLower(url);
   String16Set url_words = String16SetFromString16(url,
       word_starts ? &word_starts->url_word_starts_ : NULL);
   String16Set title_words = String16SetFromString16(row.title(),
       word_starts ? &word_starts->title_word_starts_ : NULL);
   String16Set words;
   std::set_union(url_words.begin(), url_words.end(),
@@ skipping 108 matching lines @@
   if (iter != history_id_word_map_.end()) {
     WordIDSet& word_id_set(iter->second);
     word_id_set.insert(word_id);
   } else {
     WordIDSet word_id_set;
     word_id_set.insert(word_id);
     history_id_word_map_[history_id] = word_id_set;
   }
 }
 
-bool URLIndexPrivateData::UpdateURL(const URLRow& row) {
+bool URLIndexPrivateData::UpdateURL(
+    const URLRow& row,
+    const std::string& languages,
+    const std::set<std::string>& scheme_whitelist) {
   // The row may or may not already be in our index. If it is not already
   // indexed and it qualifies then it gets indexed. If it is already
   // indexed and still qualifies then it gets updated, otherwise it
   // is deleted from the index.
   bool row_was_updated = false;
   URLID row_id = row.id();
   HistoryInfoMap::iterator row_pos = history_info_map_.find(row_id);
   if (row_pos == history_info_map_.end()) {
     // This new row should be indexed if it qualifies.
     URLRow new_row(row);
     new_row.set_id(row_id);
-    row_was_updated =
-        RowQualifiesAsSignificant(new_row, base::Time()) && IndexRow(new_row);
+    row_was_updated = RowQualifiesAsSignificant(new_row, base::Time()) &&
+                      IndexRow(new_row, languages, scheme_whitelist);
   } else if (RowQualifiesAsSignificant(row, base::Time())) {
     // This indexed row still qualifies and will be re-indexed.
     // The url won't have changed but the title, visit count, etc.
     // might have changed.
     URLRow& row_to_update = row_pos->second;
     bool title_updated = row_to_update.title() != row.title();
     if (row_to_update.visit_count() != row.visit_count() ||
         row_to_update.typed_count() != row.typed_count() ||
         row_to_update.last_visit() != row.last_visit() || title_updated) {
       row_to_update.set_visit_count(row.visit_count());
       row_to_update.set_typed_count(row.typed_count());
       row_to_update.set_last_visit(row.last_visit());
       // While the URL is guaranteed to remain stable, the title may have
       // changed. If so, then update the index with the changed words.
       if (title_updated) {
         // Clear all words associated with this row and re-index both the
         // URL and title.
         RemoveRowWordsFromIndex(row_to_update);
         row_to_update.set_title(row.title());
         RowWordStarts word_starts;
-        AddRowWordsToIndex(row_to_update, &word_starts);
+        AddRowWordsToIndex(row_to_update, &word_starts, languages);
         word_starts_map_[row_id] = word_starts;
       }
       row_was_updated = true;
     }
   } else {
     // This indexed row no longer qualifies and will be de-indexed by
     // clearing all words associated with this row.
     RemoveRowFromIndex(row);
     row_was_updated = true;
   }
@@ skipping 21 matching lines @@
       history_info_map_.begin(),
       history_info_map_.end(),
       HistoryInfoMapItemHasURL(url));
   if (pos == history_info_map_.end())
     return false;
   RemoveRowFromIndex(pos->second);
   search_term_cache_.clear();  // This invalidates the cache.
   return true;
 }
 
-bool URLIndexPrivateData::URLSchemeIsWhitelisted(const GURL& gurl) const {
-  return scheme_whitelist_.find(gurl.scheme()) != scheme_whitelist_.end();
-}
-
 // URLIndexPrivateData::HistoryItemFactorGreater -------------------------------
 
 URLIndexPrivateData::HistoryItemFactorGreater::HistoryItemFactorGreater(
     const HistoryInfoMap& history_info_map)
     : history_info_map_(history_info_map) {
 }
 
 URLIndexPrivateData::HistoryItemFactorGreater::~HistoryItemFactorGreater() {}
 
 bool URLIndexPrivateData::HistoryItemFactorGreater::operator()(
@@ skipping 498 matching lines @@
       std::set_intersection(word_id_set.begin(), word_id_set.end(),
                             char_word_id_set.begin(), char_word_id_set.end(),
                             std::inserter(new_word_id_set,
                                           new_word_id_set.begin()));
       word_id_set.swap(new_word_id_set);
     }
   }
   return word_id_set;
 }
 
-// static
-void URLIndexPrivateData::InitializeSchemeWhitelist(
-    std::set<std::string>* whitelist) {
-  DCHECK(whitelist);
-  whitelist->insert(std::string(chrome::kAboutScheme));
-  whitelist->insert(std::string(chrome::kChromeUIScheme));
-  whitelist->insert(std::string(chrome::kFileScheme));
-  whitelist->insert(std::string(chrome::kFtpScheme));
-  whitelist->insert(std::string(chrome::kHttpScheme));
-  whitelist->insert(std::string(chrome::kHttpsScheme));
-  whitelist->insert(std::string(chrome::kMailToScheme));
-}
-
 // Cache Saving ----------------------------------------------------------------
 
+// static
+void URLIndexPrivateData::WritePrivateDataToCacheFileTask(
+    scoped_refptr<URLIndexPrivateData> private_data,
+    const FilePath& file_path,
+    scoped_refptr<RefCountedBool> succeeded) {
+  DCHECK(private_data.get());
+  DCHECK(!file_path.empty());
+  succeeded->set_value(private_data->SaveToFile(file_path));
+}
+
 bool URLIndexPrivateData::SaveToFile(const FilePath& file_path) {
-  // TODO(mrossetti): Move File IO to another thread.
-  base::ThreadRestrictions::ScopedAllowIO allow_io;
   base::TimeTicks beginning_time = base::TimeTicks::Now();
   InMemoryURLIndexCacheItem index_cache;
   SavePrivateData(&index_cache);
   std::string data;
   if (!index_cache.SerializeToString(&data)) {
     LOG(WARNING) << "Failed to serialize the InMemoryURLIndex cache.";
     return false;
   }
 
   int size = data.size();
@@ skipping 87 matching lines @@
   if (history_info_map_.empty())
     return;
   HistoryInfoMapItem* map_item = cache->mutable_history_info_map();
   map_item->set_item_count(history_info_map_.size());
   for (HistoryInfoMap::const_iterator iter = history_info_map_.begin();
        iter != history_info_map_.end(); ++iter) {
     HistoryInfoMapEntry* map_entry = map_item->add_history_info_map_entry();
     map_entry->set_history_id(iter->first);
     const URLRow& url_row(iter->second);
     // Note: We only save information that contributes to the index so there
-    // is no need to save search_term_cache_ (not persistent),
-    // languages_, etc.
+    // is no need to save search_term_cache_ (not persistent).
     map_entry->set_visit_count(url_row.visit_count());
     map_entry->set_typed_count(url_row.typed_count());
     map_entry->set_last_visit(url_row.last_visit().ToInternalValue());
     map_entry->set_url(url_row.url().spec());
     map_entry->set_title(UTF16ToUTF8(url_row.title()));
   }
 }
 
 void URLIndexPrivateData::SaveWordStartsMap(
     InMemoryURLIndexCacheItem* cache) const {
@@ skipping 18 matching lines @@
          i != word_starts.url_word_starts_.end(); ++i)
       map_entry->add_url_word_starts(*i);
     for (WordStarts::const_iterator i = word_starts.title_word_starts_.begin();
          i != word_starts.title_word_starts_.end(); ++i)
       map_entry->add_title_word_starts(*i);
   }
 }
 
 // Cache Restoring -------------------------------------------------------------
 
-bool URLIndexPrivateData::RestoreFromFile(const FilePath& file_path) {
-  // TODO(mrossetti): Figure out how to determine if the cache is up-to-date.
-  // That is: ensure that the database has not been modified since the cache
-  // was last saved. DB file modification date is inadequate. There are no
-  // SQLite table checksums automatically stored.
-  Clear();  // Start with a clean slate.
+// static
+void URLIndexPrivateData::RestoreFromFileTask(
+    const FilePath& file_path,
+    scoped_refptr<URLIndexPrivateData> private_data,
+    std::string languages) {
+  private_data = URLIndexPrivateData::RestoreFromFile(file_path, languages);
+}
 
-  // FIXME(mrossetti): Move File IO to another thread.
-  base::ThreadRestrictions::ScopedAllowIO allow_io;
+// static
+scoped_refptr<URLIndexPrivateData> URLIndexPrivateData::RestoreFromFile(
+    const FilePath& file_path,
+    const std::string& languages) {
   base::TimeTicks beginning_time = base::TimeTicks::Now();
   if (!file_util::PathExists(file_path))
-    return false;
+    return NULL;
   std::string data;
   // If there is no cache file then simply give up. This will cause us to
   // attempt to rebuild from the history database.
   if (!file_util::ReadFileToString(file_path, &data))
-    return false;
+    return NULL;
 
+  scoped_refptr<URLIndexPrivateData> restored_data(new URLIndexPrivateData);
   InMemoryURLIndexCacheItem index_cache;
   if (!index_cache.ParseFromArray(data.c_str(), data.size())) {
-    LOG(WARNING) << "Failed to parse InMemoryURLIndex cache data read from "
+    LOG(WARNING) << "Failed to parse URLIndexPrivateData cache data read from "
                  << file_path.value();
-    return false;
+    return restored_data;
   }
 
-  if (!RestorePrivateData(index_cache)) {
-    Clear();  // Back to square one -- must build from scratch.
-    return false;
-  }
+  if (!restored_data->RestorePrivateData(index_cache, languages))
+    return NULL;
 
   UMA_HISTOGRAM_TIMES("History.InMemoryURLIndexRestoreCacheTime",
                       base::TimeTicks::Now() - beginning_time);
   UMA_HISTOGRAM_COUNTS("History.InMemoryURLHistoryItems",
-                       history_id_word_map_.size());
+                       restored_data->history_id_word_map_.size());
   UMA_HISTOGRAM_COUNTS("History.InMemoryURLCacheSize", data.size());
-  UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLWords", word_map_.size());
-  UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLChars", char_word_map_.size());
-  return true;
+  UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLWords",
+                             restored_data->word_map_.size());
+  UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLChars",
+                             restored_data->char_word_map_.size());
+  if (restored_data->Empty())
+    return NULL;  // 'No data' is the same as a failed reload.
+  return restored_data;
 }
 
 // static
-URLIndexPrivateData* URLIndexPrivateData::RebuildFromHistory(
-    HistoryDatabase* history_db) {
+scoped_refptr<URLIndexPrivateData> URLIndexPrivateData::RebuildFromHistory(
+    HistoryDatabase* history_db,
+    const std::string& languages,
+    const std::set<std::string>& scheme_whitelist) {
   if (!history_db)
     return NULL;
 
   base::TimeTicks beginning_time = base::TimeTicks::Now();
 
-  scoped_ptr<URLIndexPrivateData> rebuilt_data(new URLIndexPrivateData);
+  scoped_refptr<URLIndexPrivateData> rebuilt_data(new URLIndexPrivateData);
   URLDatabase::URLEnumerator history_enum;
   if (!history_db->InitURLEnumeratorForSignificant(&history_enum))
     return NULL;
   for (URLRow row; history_enum.GetNextURL(&row); )
-    rebuilt_data->IndexRow(row);
+    rebuilt_data->IndexRow(row, languages, scheme_whitelist);
 
   UMA_HISTOGRAM_TIMES("History.InMemoryURLIndexingTime",
                       base::TimeTicks::Now() - beginning_time);
   UMA_HISTOGRAM_COUNTS("History.InMemoryURLHistoryItems",
                        rebuilt_data->history_id_word_map_.size());
   UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLWords",
                              rebuilt_data->word_map_.size());
   UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLChars",
                              rebuilt_data->char_word_map_.size());
-  return rebuilt_data.release();
+  return rebuilt_data;
 }
 
 bool URLIndexPrivateData::RestorePrivateData(
-    const InMemoryURLIndexCacheItem& cache) {
+    const InMemoryURLIndexCacheItem& cache,
+    const std::string& languages) {
   if (cache.has_version())
     restored_cache_version_ = cache.version();
   return RestoreWordList(cache) && RestoreWordMap(cache) &&
       RestoreCharWordMap(cache) && RestoreWordIDHistoryMap(cache) &&
-      RestoreHistoryInfoMap(cache) && RestoreWordStartsMap(cache);
+      RestoreHistoryInfoMap(cache) && RestoreWordStartsMap(cache, languages);
 }
 
 bool URLIndexPrivateData::RestoreWordList(
     const InMemoryURLIndexCacheItem& cache) {
   if (!cache.has_word_list())
     return false;
   const WordListItem& list_item(cache.word_list());
   uint32 expected_item_count = list_item.word_count();
   uint32 actual_item_count = list_item.word_size();
   if (actual_item_count == 0 || actual_item_count != expected_item_count)
@@ skipping 101 matching lines @@
     if (iter->has_title()) {
       string16 title(UTF8ToUTF16(iter->title()));
       url_row.set_title(title);
     }
     history_info_map_[history_id] = url_row;
   }
   return true;
 }
 
 bool URLIndexPrivateData::RestoreWordStartsMap(
-    const InMemoryURLIndexCacheItem& cache) {
+    const InMemoryURLIndexCacheItem& cache,
+    const std::string& languages) {
   // Note that this function must be called after RestoreHistoryInfoMap() has
   // been run as the word starts may have to be recalculated from the urls and
   // page titles.
   if (cache.has_word_starts_map()) {
     const WordStartsMapItem& list_item(cache.word_starts_map());
     uint32 expected_item_count = list_item.item_count();
     uint32 actual_item_count = list_item.word_starts_map_entry_size();
     if (actual_item_count == 0 || actual_item_count != expected_item_count)
       return false;
     const RepeatedPtrField<WordStartsMapEntry>&
@@ skipping 14 matching lines @@
         word_starts.title_word_starts_.push_back(*jiter);
       word_starts_map_[history_id] = word_starts;
     }
   } else {
     // Since the cache did not contain any word starts we must rebuild then from
     // the URL and page titles.
     for (HistoryInfoMap::const_iterator iter = history_info_map_.begin();
          iter != history_info_map_.end(); ++iter) {
       RowWordStarts word_starts;
       const URLRow& row(iter->second);
-      string16 url(net::FormatUrl(row.url(), languages_,
+      string16 url(net::FormatUrl(row.url(), languages,
           net::kFormatUrlOmitUsernamePassword,
           net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS,
           NULL, NULL, NULL));
       url = base::i18n::ToLower(url);
       String16VectorFromString16(url, false, &word_starts.url_word_starts_);
       String16VectorFromString16(
           row.title(), false, &word_starts.title_word_starts_);
       word_starts_map_[iter->first] = word_starts;
     }
   }
   return true;
 }
 
+// static
+bool URLIndexPrivateData::URLSchemeIsWhitelisted(
+    const GURL& gurl,
+    const std::set<std::string>& whitelist) {
+  return whitelist.find(gurl.scheme()) != whitelist.end();
+}
+
 }  // namespace history