LevelDB storage for data reduction proxy to store data usage stats.

components/data_reduction_proxy/core/browser/data_usage_storage_helper.cc

Index: components/data_reduction_proxy/core/browser/data_usage_storage_helper.cc
diff --git a/components/data_reduction_proxy/core/browser/data_usage_storage_helper.cc b/components/data_reduction_proxy/core/browser/data_usage_storage_helper.cc
new file mode 100644
index 0000000000000000000000000000000000000000..1ea3ee32ef4c620a5d16f8726d77fffd2fd1d45b
--- /dev/null
+++ b/components/data_reduction_proxy/core/browser/data_usage_storage_helper.cc
@@ -0,0 +1,174 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Each |DataUsageBucket| corresponds to data usage for an interval of
+// |kDataUsageBucketLengthMins| minutes. We store data usage for the past
+// |kNumDataUsageBuckets| buckets. Buckets are maintained as a circular array
+// with indexes from 0 to (kNumDataUsageBuckets - 1). To store the circular
+// array in a key-value store, we convert each index to a unique key. The latest
+// bucket persisted to DB overwrites the oldest.
+#include "components/data_reduction_proxy/core/browser/data_usage_storage_helper.h"
+
+#include <stdlib.h>
+
+#include <algorithm>
+#include <map>
+#include <string>
+#include <utility>
+
+#include "base/strings/stringprintf.h"
+#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_store.h"
+#include "components/data_reduction_proxy/proto/store.pb.h"
+
+namespace {
+const char kCurrentBucketIndexKey[] = "current_bucket_index";
+const char kBucketKeyPrefix[] = "data_usage_bucket:";
+
+const int kMinsInHour = 60;

[bengr, 2015/06/25 23:22:55]

kMinutesInHour

[Not at Google. Contact bengr, 2015/06/29 20:53:27]

Done.

+const int kMinutesInDay = 24 * kMinsInHour;
+
+// Time interval for each DataUsageBucket.
+const int kDataUsageBucketLengthMins = 15;

[bengr, 2015/06/25 23:22:55]

kDataUsageBucketLengthInMinutes

[Not at Google. Contact bengr, 2015/06/29 20:53:27]

Done.

+static_assert(kMinsInHour % kDataUsageBucketLengthMins == 0,

[bengr, 2015/06/25 23:22:55]

kDataUsageBucketLengthMins should also be > 0.

[Not at Google. Contact bengr, 2015/06/29 20:53:27]

Done. Although someone wanting to specify a negative value here will not be
stopped by a static assert:)

+              "kDataUsageBucketLengthMins must be a factor of kMinsInHour");
+
+// Number of days for which to maintain data usage history.
+const int kDataUsageHistoryNumDays = 60;
+
+// Total number of buckets persisted to DB.
+const int kNumDataUsageBuckets =
+    kDataUsageHistoryNumDays * kMinutesInDay / kDataUsageBucketLengthMins;
+
+std::string DbKeyForBucketIndex(int index) {
+  DCHECK_GE(index, 0);
+  DCHECK_LT(index, kNumDataUsageBuckets);
+
+  return base::StringPrintf("%s%d", kBucketKeyPrefix, index);
+}
+
+base::Time BucketLowerBoundary(base::Time time) {
+  base::Time::Exploded exploded;
+  time.UTCExplode(&exploded);
+  exploded.minute -= exploded.minute % kDataUsageBucketLengthMins;
+
+  return base::Time::FromUTCExploded(exploded);
+}
+
+}  // namespace
+
+namespace data_reduction_proxy {
+
+DataUsageStorageHelper::DataUsageStorageHelper(
+    scoped_refptr<DataReductionProxyStore> db)
+    : db_(db), current_bucket_index_(-1) {
+  thread_checker_.DetachFromThread();
+}
+
+void DataUsageStorageHelper::LoadCurrentDataUsageBucket(

[michaeln, 2015/06/26 20:15:01]

returning a DataUsageBucket might make more sense for this method

[Not at Google. Contact bengr, 2015/06/29 20:53:28]

Done.

+    DataUsageBucket* current) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(current);
+
+  std::string current_index_string;
+  DataReductionProxyStore::Status index_read_status =
+      db_->Get(kCurrentBucketIndexKey, &current_index_string);
+
+  if (index_read_status == DataReductionProxyStore::Status::OK)
+    current_bucket_index_ = atoi(current_index_string.c_str());
+  else
+    current_bucket_index_ = 0;
+
+  DCHECK_GE(current_bucket_index_, 0);
+  DCHECK_LT(current_bucket_index_, kNumDataUsageBuckets);
+
+  std::string bucket_key = DbKeyForBucketIndex(current_bucket_index_);
+  std::string bucket_as_string;
+  DataReductionProxyStore::Status bucket_read_status =
+      db_->Get(bucket_key, &bucket_as_string);
+  if ((bucket_read_status != DataReductionProxyStore::Status::OK &&
+       bucket_read_status != DataReductionProxyStore::NOT_FOUND)) {
+    LOG(WARNING) << "Failed to read data usage bucket from LevelDB: "
+                 << bucket_read_status;
+  }
+
+  if (bucket_read_status == DataReductionProxyStore::Status::OK) {
+    DCHECK(current->ParseFromString(bucket_as_string));

[michaeln, 2015/06/26 20:15:01]

why is |current| a parameter to this method since its only used in a DCHECK?

[Not at Google. Contact bengr, 2015/06/29 20:53:27]

ParseFromString(..) populates |current|.

[michaeln, 2015/06/29 23:19:30]

DCHECKS are compiled out of release builds

[Not at Google. Contact bengr, 2015/07/01 17:13:48]

Doh! Thanks:)

+    current_bucket_last_updated_ =
+        base::Time::FromInternalValue(current->last_updated_timestamp());
+  }
+}
+
+void DataUsageStorageHelper::StoreCurrentDataUsageBucket(
+    const DataUsageBucket* current) {

[michaeln, 2015/06/26 20:15:01]

const ref might make more sense for this arg

[Not at Google. Contact bengr, 2015/06/29 20:53:28]

Called from a different thread. So const DataUsageBucket* or const
scoped_refptr<>& seem to make more sense. Otherwise  the other thread will have
to own it. With either DataUsageBucket* or const scoped_refptr<>&, ref counting
avoids having to explicitly deal with ownership.

[michaeln, 2015/06/29 23:19:30]

Not sure const T* makes any more or less sense then const T&? DataUsageBucket is
not a refcounted class so the discusion of refcounting doesn't seem to make
sense here.

[Not at Google. Contact bengr, 2015/07/01 17:13:48]

Changed to const ref.

I meant "T*" seemed easier than "const T&" since "T*" allows pointer to be bound
to callback. 

Wasn't sure where the object would live if I used "const T&". Now I have a
DBService which can hold this reference on the same thread which allows using
"const T&".

+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(current);

[bengr, 2015/06/25 23:22:55]

No need to DCHECK the pointer. You'll know quickly enough.

[Not at Google. Contact bengr, 2015/06/29 20:53:28]

This is more to set expectations for someone reading the code. For example,
someone might come along and put a null check here. 

+  DCHECK(current_bucket_index_ >= 0 &&
+         current_bucket_index_ < kNumDataUsageBuckets);
+
+  // If current bucket does not have any information, we skip writing to DB.
+  if (!current->has_last_updated_timestamp())
+    return;
+
+  int prev_current_bucket_index = current_bucket_index_;
+  // We might have skipped saving buckets because Chrome was not being used.
+  // Write empty buckets to those slots to overwrite outdated information.
+  base::Time last_updated =
+      base::Time::FromInternalValue(current->last_updated_timestamp());
+  std::map<std::string, std::string> buckets_to_save;
+  int num_buckets_since_last_saved = NumBucketsSinceLastSaved(last_updated);
+  for (int i = 0; i < num_buckets_since_last_saved; i++) {

[bengr, 2015/06/25 23:22:55]

++i

[Not at Google. Contact bengr, 2015/06/29 20:53:28]

Done.

+    AddBucketToMap(new DataUsageBucket(), &buckets_to_save);

[michaeln, 2015/06/26 20:15:01]

does this leak buckets?

[Not at Google. Contact bengr, 2015/06/29 20:53:27]

Good catch. Fixed.

+
+    current_bucket_index_++;
+    DCHECK(current_bucket_index_ <= kNumDataUsageBuckets);
+    if (current_bucket_index_ == kNumDataUsageBuckets)
+      current_bucket_index_ = 0;
+  }
+
+  AddBucketToMap(current, &buckets_to_save);
+
+  std::stringstream current_index_string;
+  current_index_string << current_bucket_index_;
+  buckets_to_save.insert(std::pair<std::string, std::string>(
+      kCurrentBucketIndexKey, current_index_string.str()));
+
+  DataReductionProxyStore::Status status = db_->Put(buckets_to_save);
+  if (status != DataReductionProxyStore::Status::OK) {
+    current_bucket_index_ = prev_current_bucket_index;
+    LOG(WARNING) << "Failed to write data usage buckets to LevelDB" << status;
+  }
+}
+
+DataUsageStorageHelper::~DataUsageStorageHelper() {
+}
+
+void DataUsageStorageHelper::AddBucketToMap(
+    const DataUsageBucket* bucket,

[michaeln, 2015/06/26 20:15:01]

const ref might make more sense for this argument

[Not at Google. Contact bengr, 2015/06/29 20:53:28]

StoreCurrentDataUsageBucket will be called from different thread which in turn
calls this. Not sure const reference to entity on another thread makes sense.

+    std::map<std::string, std::string>* map) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  std::string bucket_key = DbKeyForBucketIndex(current_bucket_index_);
+
+  std::string bucket_value;
+  bool success = bucket->SerializeToString(&bucket_value);
+  DCHECK(success);
+
+  map->insert(std::pair<std::string, std::string>(bucket_key, bucket_value));
+}
+
+int DataUsageStorageHelper::NumBucketsSinceLastSaved(
+    base::Time new_last_updated_timestamp) const {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (current_bucket_last_updated_.is_null())
+    return 0;
+
+  base::TimeDelta time_delta =
+      BucketLowerBoundary(new_last_updated_timestamp) -
+      BucketLowerBoundary(current_bucket_last_updated_);
+  int num_buckets_skipped = time_delta.InMinutes() / kDataUsageBucketLengthMins;
+  return std::min(num_buckets_skipped, kNumDataUsageBuckets - 1);
+}
+
+}  // namespace data_reduction_proxy