Add daily quota for precache

components/precache/core/precache_fetcher.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_fetcher.h"
 
 #include <algorithm>
 #include <limits>
 #include <utility>
 #include <vector>
 
 #include "base/base64.h"
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/containers/hash_tables.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/sha1.h"
 #include "base/strings/string_piece.h"
 #include "base/task_runner_util.h"
 #include "components/precache/core/precache_database.h"
 #include "components/precache/core/precache_switches.h"
 #include "components/precache/core/proto/precache.pb.h"
+#include "components/precache/core/proto/quota.pb.h"
 #include "components/precache/core/proto/unfinished_work.pb.h"
 #include "net/base/completion_callback.h"
 #include "net/base/escape.h"
 #include "net/base/io_buffer.h"
 #include "net/base/load_flags.h"
 #include "net/base/net_errors.h"
 #include "net/base/url_util.h"
 #include "net/http/http_response_headers.h"
 #include "net/url_request/url_fetcher_response_writer.h"
 #include "net/url_request/url_request_context_getter.h"
@@ skipping 157 matching lines @@
           GetResourceURLBase64Hash(unused_urls)));
     } else {
       hosts_info.push_back(
           ManifestHostInfo(PrecacheReferrerHostEntry::kInvalidId, host,
                            std::string(), std::string()));
     }
   }
   return hosts_info;
 }
 
+PrecacheQuota RetrieveQuotaInfo(
+    const base::WeakPtr<PrecacheDatabase>& precache_database) {
+  PrecacheQuota quota;
+  if (precache_database) {
+    quota = precache_database->GetQuota();
+  }
+  return quota;
+}
+
+// Returns true if the |quota| time has expired.
+bool IsQuotaTimeExpired(const PrecacheQuota& quota,
+                        const base::Time& time_now) {
+  // Quota expires by the end of the day specified in start time.

[sclittle, 2016/09/22 21:36:55]

nit: As implemented below, I don't think this comment is exactly right - the
quota expires exactly 1 day after start_time, doesn't it? Not at the end of the
day?

Could you either change the code or the comment?

[Raj, 2016/09/23 01:32:41]

Done.

+  base::Time start_time = base::Time::FromInternalValue(quota.start_time());
+  return start_time > time_now ||
+         start_time + base::TimeDelta::FromDays(1) < time_now;
+}
+
 }  // namespace
 
 PrecacheFetcher::Fetcher::Fetcher(
     net::URLRequestContextGetter* request_context,
     const GURL& url,
     const std::string& referrer,
     const base::Callback<void(const Fetcher&)>& callback,
     bool is_resource_request,
     size_t max_bytes)
     : request_context_(request_context),
@@ skipping 242 matching lines @@
   pool_.Add(base::MakeUnique<Fetcher>(
       request_context_.get(), config_url, std::string(),
       base::Bind(&PrecacheFetcher::OnConfigFetchComplete, AsWeakPtr()),
       false /* is_resource_request */, std::numeric_limits<int32_t>::max()));
 }
 
 void PrecacheFetcher::StartNextResourceFetch() {
   DCHECK(unfinished_work_->has_config_settings());
   while (!resources_to_fetch_.empty() && pool_.IsAvailable()) {
     const auto& resource = resources_to_fetch_.front();
-    const size_t max_bytes =
+    const size_t max_bytes = std::min(
+        quota_.remaining(),
         std::min(unfinished_work_->config_settings().max_bytes_per_resource(),
                  unfinished_work_->config_settings().max_bytes_total() -
-                     unfinished_work_->total_bytes());
+                     unfinished_work_->total_bytes()));
     VLOG(3) << "Fetching " << resource.first << " " << resource.second;
     pool_.Add(base::MakeUnique<Fetcher>(
         request_context_.get(), resource.first, resource.second,
         base::Bind(&PrecacheFetcher::OnResourceFetchComplete, AsWeakPtr()),
         true /* is_resource_request */, max_bytes));
 
     resources_to_fetch_.pop_front();
   }
 }
 
@@ skipping 16 matching lines @@
     size_t remaining_manifest_urls_to_fetch,
     size_t remaining_resource_urls_to_fetch) {
   RecordCompletionStatistics(*unfinished_work_,
                              remaining_manifest_urls_to_fetch,
                              remaining_resource_urls_to_fetch);
   precache_delegate_->OnDone();
 }
 
 void PrecacheFetcher::StartNextFetch() {
   DCHECK(unfinished_work_->has_config_settings());
-  // If over the precache total size cap, then stop prefetching.
-  if (unfinished_work_->total_bytes() >
-      unfinished_work_->config_settings().max_bytes_total()) {
+
+  // If over the precache total size cap or daily quota, then stop prefetching.
+  if ((unfinished_work_->total_bytes() >
+       unfinished_work_->config_settings().max_bytes_total()) ||
+      quota_.remaining() == 0) {
     size_t pending_manifests_in_pool = 0;
     size_t pending_resources_in_pool = 0;
     for (const auto& element_pair : pool_.elements()) {
       const Fetcher* fetcher = element_pair.first;
       if (fetcher->is_resource_request())
         pending_resources_in_pool++;
       else if (fetcher->url() != config_url_)
         pending_manifests_in_pool++;
     }
     pool_.DeleteAll();
@@ skipping 93 matching lines @@
           manifest.manifest_url, "manifest",
           std::to_string(manifest.manifest_id));
       manifest.manifest_url = net::AppendOrReplaceQueryParameter(
           manifest.manifest_url, "used_resources", manifest.used_url_hash);
       manifest.manifest_url = net::AppendOrReplaceQueryParameter(
           manifest.manifest_url, "unused_resources", manifest.unused_url_hash);
       DCHECK(manifest.manifest_url.is_valid());
     }
   }
   unfinished_work_->set_num_manifest_urls(top_hosts_to_fetch_.size());
+
+  PostTaskAndReplyWithResult(
+      db_task_runner_.get(), FROM_HERE,
+      base::Bind(&RetrieveQuotaInfo, precache_database_),
+      base::Bind(&PrecacheFetcher::OnQuotaInfoRetrieved, AsWeakPtr()));
+}
+
+void PrecacheFetcher::OnQuotaInfoRetrieved(const PrecacheQuota& quota) {
+  quota_ = quota;
+  base::Time time_now = base::Time::Now();
+  if (IsQuotaTimeExpired(quota_, time_now)) {
+    // This is a new day. Update daily quota, that starts today and expires by
+    // end of today.
+    quota_.set_start_time(time_now.LocalMidnight().ToInternalValue());
+    quota_.set_remaining(
+        unfinished_work_->config_settings().daily_quota_total());
+    db_task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&PrecacheDatabase::SaveQuota, precache_database_, quota_));
+  }
   StartNextFetch();
 }
 
 ManifestHostInfo::ManifestHostInfo(int64_t manifest_id,
                                    const std::string& hostname,
                                    const std::string& used_url_hash,
                                    const std::string& unused_url_hash)
     : manifest_id(manifest_id),
       hostname(hostname),
       used_url_hash(used_url_hash),
@@ skipping 49 matching lines @@
                  source.was_cached(), source.response_bytes()));
 
   pool_.Delete(source);
 
   // The resource has already been put in the cache during the fetch process, so
   // nothing more needs to be done for the resource.
   StartNextFetch();
 }
 
 void PrecacheFetcher::UpdateStats(int64_t response_bytes,
                                   int64_t network_response_bytes) {

[sclittle, 2016/09/22 21:36:55]

nit: Could you add DCHECKs here that |response_bytes| and
|network_response_bytes| are non-negative?

[Raj, 2016/09/23 01:32:41]

Done.

   unfinished_work_->set_total_bytes(
       unfinished_work_->total_bytes() + response_bytes);
   unfinished_work_->set_network_bytes(
       unfinished_work_->network_bytes() + network_response_bytes);
+
+  if (!IsQuotaTimeExpired(quota_, base::Time::Now())) {
+    int64_t remaining =
+        static_cast<int64_t>(quota_.remaining()) - network_response_bytes;

[sclittle, 2016/09/22 21:36:55]

nit: Instead of doing all this casting from unsigned to signed here, which will
cause bugs if quota_.remaining() is between max signed int64 and max uint64, I
think it'd probably be cleaner to restructure this code here to not allow
|remaining| to go negative since network_response_bytes is guaranteed to be
non-negative, e.g.:

uint64_t used_bytes = static_cast<uint64_t>(network_response_bytes);
quota_.set_remaining(used_bytes > quota_.remaining() ? 0U : quota_.remaining() -
used_bytes);

Alternatively, you could just change PrecacheQuota.remaining into a signed
int64. I don't think there's much point in keeping it an unsigned int64 if it
doesn't support numbers larger than max signed int64.

[Raj, 2016/09/23 01:32:41]

Done.

+    if (remaining < 0)
+      remaining = 0;
+    quota_.set_remaining(static_cast<uint64_t>(remaining));
+    db_task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&PrecacheDatabase::SaveQuota, precache_database_, quota_));
+  }
 }
 
 }  // namespace precache