Precache tracking database

components/precache/core/precache_database.cc

Index: components/precache/core/precache_database.cc
diff --git a/components/precache/core/precache_database.cc b/components/precache/core/precache_database.cc
new file mode 100644
index 0000000000000000000000000000000000000000..239b62ea1545fe74eccc7dd00fa84f56972c4811
--- /dev/null
+++ b/components/precache/core/precache_database.cc
@@ -0,0 +1,191 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/precache/core/precache_database.h"
+
+#include "base/metrics/histogram.h"
+#include "base/time/time.h"
+#include "components/precache/core/precache_statistics_table.h"
+#include "components/precache/core/precache_url_table.h"
+#include "sql/connection.h"
+#include "url/gurl.h"
+
+namespace {
+
+// The number of days old that an entry in the precache URL table can be before
+// it is considered "old" and is removed from the table.
+const int64 kPrecacheHistoryExpiryPeriodDays = 60;
+
+void RecordSingleDayPrecacheUMA(
+    const precache::PrecacheStatisticsTable::PrecacheStatistics& stats) {
+  UMA_HISTOGRAM_COUNTS("Precache.DailyDownloadedPrecacheMotivatedKB",
+                       stats.downloaded_precache_motivated_bytes / 1024);
+  UMA_HISTOGRAM_COUNTS("Precache.DailyDownloadedNonPrecacheKB",
+                       stats.downloaded_non_precache_bytes / 1024);
+  UMA_HISTOGRAM_COUNTS("Precache.DailyDownloadedNonPrecacheKB.Cellular",
+                       stats.downloaded_non_precache_bytes_cellular / 1024);
+  UMA_HISTOGRAM_COUNTS("Precache.DailySavedKB", stats.saved_bytes / 1024);
+  UMA_HISTOGRAM_COUNTS("Precache.DailySavedKB.Cellular",
+                       stats.saved_bytes_cellular / 1024);
+
+  if (stats.saved_bytes + stats.downloaded_non_precache_bytes > 0) {
+    UMA_HISTOGRAM_PERCENTAGE(
+        "Precache.DailySavingsPercentage",
+        stats.saved_bytes * 100.0 /
+            (stats.saved_bytes + stats.downloaded_non_precache_bytes));
+  }
+
+  if (stats.saved_bytes_cellular +
+          stats.downloaded_non_precache_bytes_cellular > 0) {
+    UMA_HISTOGRAM_PERCENTAGE(
+        "Precache.DailySavingsPercentage.Cellular",
+        stats.saved_bytes_cellular * 100.0 /
+            (stats.saved_bytes_cellular +
+             stats.downloaded_non_precache_bytes_cellular));
+  }
+}
+
+}  // namespace
+
+namespace precache {
+
+PrecacheDatabase::PrecacheDatabase()
+    : precache_url_table_(new PrecacheURLTable()),
+      precache_statistics_table_(new PrecacheStatisticsTable()) {
+  // A PrecacheDatabase can be constructed on any thread.
+  DetachFromThread();
+}
+
+PrecacheDatabase::~PrecacheDatabase() {
+  // Since the PrecacheDatabase is refcounted, it will only be deleted if there
+  // are no references remaining to it, meaning that it is not in use. Thus, it
+  // is safe to delete it, regardless of what thread we are on.
+  DetachFromThread();
+}
+
+void PrecacheDatabase::Init(scoped_ptr<sql::Connection> db) {
+  DCHECK(CalledOnValidThread());
+  DCHECK(!db_);  // Init must only be called once.
+  DCHECK(db);    // |db| must not be NULL.
+
+  db_.reset(db.release());

[davidben, 2013/11/06 23:50:53]

Nit: You can write "db_ = db.Pass();" which is a slightly safer pattern.
(Emulation of move semantics from C++11.)

[sclittle, 2013/11/09 01:07:58]

Done.

+  if (!IsDatabaseAccessible()) {
+    // Don't initialize the URL table or statistics table if unable to access
+    // the database.
+    return;
+  }
+
+  precache_url_table_->Init(db_.get());
+  precache_statistics_table_->Init(db_.get());
+}
+
+void PrecacheDatabase::ReportAndDeleteOldStats(const base::Time& end_date) {
+  DCHECK(CalledOnValidThread());
+  DCHECK(db_);
+
+  if (!IsDatabaseAccessible()) {
+    // Do nothing if unable to access the database.
+    return;
+  }
+
+  // Delete old precache history that has expired.
+  precache_url_table_->DeleteAllPrecachedUntil(
+      end_date - base::TimeDelta::FromDays(kPrecacheHistoryExpiryPeriodDays));
+
+  PrecacheStatisticsTable::PrecacheStatisticsMap stats_map;
+  precache_statistics_table_->GetAllStatsUntil(end_date, &stats_map);
+
+  // Report UMA for every row of old statistics in the statistics table. There
+  // won't be any rows in the statistics table for days when nothing was fetched
+  // or precached.
+  for (PrecacheStatisticsTable::PrecacheStatisticsMap::const_iterator it =
+           stats_map.begin();
+       it != stats_map.end(); ++it) {
+    RecordSingleDayPrecacheUMA(it->second);
+  }
+
+  precache_statistics_table_->DeleteAllStatsUntil(end_date);
+}
+
+void PrecacheDatabase::RecordURLFetched(const GURL& url,
+                                        const base::Time& fetch_time,
+                                        int64 size, bool was_cached,
+                                        bool is_precache,
+                                        bool is_connection_cellular) {
+  DCHECK(CalledOnValidThread());
+  DCHECK(db_);
+
+  if (!IsDatabaseAccessible()) {
+    // Don't track anything if unable to access the database.
+    return;
+  }
+
+  if (is_precache) {
+    if (was_cached && !precache_url_table_->HasURL(url)) {
+      // Since the precache came from the cache, and there's no entry in the
+      // URL table for the URL, this means that the resource was already in the
+      // cache because of user browsing. Thus, this precache had no effect, so
+      // ignore it.
+      return;
+    }
+
+    if (!was_cached) {
+      // The precache only counts as overhead if it was downloaded over the
+      // network.
+      PrecacheStatisticsTable::PrecacheStatistics stats;
+      stats.downloaded_precache_motivated_bytes = size;
+      precache_statistics_table_->IncreaseDailyStats(fetch_time, stats);
+    }
+
+    // Use the URL table to keep track of URLs that are in the cache thanks to
+    // precaching. If a row for the URL already exists, than update the
+    // timestamp to |fetch_time|.
+    precache_url_table_->AddURL(url, fetch_time);
+    return;
+  }
+
+  if (!was_cached) {
+    // The fetch was served over the network during user browsing, so count it
+    // as downloaded non-precache bytes.
+    PrecacheStatisticsTable::PrecacheStatistics stats;
+    stats.downloaded_non_precache_bytes = size;
+    if (is_connection_cellular) {
+      stats.downloaded_non_precache_bytes_cellular = size;
+    }
+    precache_statistics_table_->IncreaseDailyStats(fetch_time, stats);
+
+    // Since the fetch was over the network during user browsing, delete any
+    // record of it having been precached from the URL table. If it had been
+    // precached, the resource must have expired.
+    precache_url_table_->DeleteURL(url);

[davidben, 2013/11/06 23:50:53]

I think there are cases where requests also just bypass the cache altogether and
not touch it, such requests with an external If-Unmodified-Since. I imagine
they're not common to care much about, and I don't know if the network stack
even exposes enough information for you there, but just FYI.

[sclittle, 2013/11/09 01:07:58]

I'm not familiar with these cases; what does it mean for a request to have an
"external" If-Unmodified-Since?

From what I can see in the HTTP 1.1 spec, it looks like a 2xx response from
these kinds of requests would still be stored in the cache, so I think this code
would handle those kinds of cases correctly. We wouldn't call RecordURLFetched
for unsuccessful responses.

[davidben, 2013/11/11 19:06:26]

By external I mean the headers originated from outside the network stack
somewhere. An XHR might have put it in there, or some plugin. The cache has
several modes when that happens. Sometimes it will still interpret the results
and write to cache but never read from it. Sometimes it will just bypass it
altogether because it doesn't know how to handle that. I believe
If-Unmodified-Since is one of those cases. So it's possible to not use the cache
on a request and still have it available for later runs.

(I don't think it's actually a big deal, and I don't know how you'd get at that
information outside the network stack. I just thought I'd mention it.)

[sclittle, 2013/11/19 04:03:41]

Interesting, thanks for the info.

+    return;
+  }
+
+  if (was_cached && precache_url_table_->HasURL(url)) {
+    // The fetch was served from the cache, and since there's an entry for this
+    // URL in the URL table, this means that the resource was served from the
+    // cache only because precaching put it there. Thus, precaching was helpful,
+    // so count the fetch as saved bytes.
+    PrecacheStatisticsTable::PrecacheStatistics stats;
+    stats.saved_bytes = size;
+    if (is_connection_cellular) {
+      stats.saved_bytes_cellular = size;
+    }
+    precache_statistics_table_->IncreaseDailyStats(fetch_time, stats);
+
+    // The current fetch would have put this resource in the cache, so precache
+    // can't take credit for subsequent fetches of this resource until the next
+    // time that the resource is precached over the network.
+    precache_url_table_->DeleteURL(url);

[davidben, 2013/11/06 23:50:53]

You could probably move the IncreaseDailyStats and DeleteURL calls out of the
two conditions and make it a little easier to follow:

   PrecacheStatisticsTable::PrecacheStatistics stats;
   if (...) {
     ... blah, compute appropriate update to stats.
   } else if (...) {
     ...
   }
   precache_statistics_table_->IncreaseDailyStats(fetch_time, stats);

   precache_url_table_->DeleteURL(url);

[sclittle, 2013/11/09 01:07:58]

Done.

+  }
+}
+
+bool PrecacheDatabase::IsDatabaseAccessible() const {
+  DCHECK(CalledOnValidThread());
+  DCHECK(db_);
+
+  return db_->is_open();
+}
+
+}  // namespace precache