Add daily quota for precache

components/precache/core/precache_fetcher.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_fetcher.h"
 
 #include <algorithm>
 #include <limits>
 #include <utility>
 #include <vector>
 
 #include "base/base64.h"
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/containers/hash_tables.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/sha1.h"
 #include "base/strings/string_piece.h"
 #include "base/task_runner_util.h"
 #include "components/precache/core/precache_database.h"
 #include "components/precache/core/precache_switches.h"
 #include "components/precache/core/proto/precache.pb.h"
+#include "components/precache/core/proto/quota.pb.h"
 #include "components/precache/core/proto/unfinished_work.pb.h"
 #include "net/base/completion_callback.h"
 #include "net/base/escape.h"
 #include "net/base/io_buffer.h"
 #include "net/base/load_flags.h"
 #include "net/base/net_errors.h"
 #include "net/base/url_util.h"
 #include "net/http/http_response_headers.h"
 #include "net/url_request/url_fetcher_response_writer.h"
 #include "net/url_request/url_request_context_getter.h"
@@ skipping 157 matching lines @@
           GetResourceURLBase64Hash(unused_urls)));
     } else {
       hosts_info.push_back(
           ManifestHostInfo(PrecacheReferrerHostEntry::kInvalidId, host,
                            std::string(), std::string()));
     }
   }
   return hosts_info;
 }
 
+PrecacheQuota RetrieveQuotaInfo(
+    const base::WeakPtr<PrecacheDatabase>& precache_database) {
+  PrecacheQuota quota;
+  if (precache_database) {
+    quota = precache_database->GetQuota();
+  }
+  return quota;
+}
+
+// Returns true if the |quota| time has expired.
+bool IsQuotaTimeExpired(const PrecacheQuota& quota,
+                        const base::Time& time_now) {
+  base::Time start_time = base::Time::FromInternalValue(quota.start_time());
+  return start_time > time_now ||
+         start_time + base::TimeDelta::FromHours(quota.expiry_hours()) <
+             time_now;
+}
+
 }  // namespace
 
 PrecacheFetcher::Fetcher::Fetcher(
     net::URLRequestContextGetter* request_context,
     const GURL& url,
     const std::string& referrer,
     const base::Callback<void(const Fetcher&)>& callback,
     bool is_resource_request,
     size_t max_bytes)
     : request_context_(request_context),
@@ skipping 242 matching lines @@
   pool_.Add(base::MakeUnique<Fetcher>(
       request_context_.get(), config_url, std::string(),
       base::Bind(&PrecacheFetcher::OnConfigFetchComplete, AsWeakPtr()),
       false /* is_resource_request */, std::numeric_limits<int32_t>::max()));
 }
 
 void PrecacheFetcher::StartNextResourceFetch() {
   DCHECK(unfinished_work_->has_config_settings());
   while (!resources_to_fetch_.empty() && pool_.IsAvailable()) {
     const auto& resource = resources_to_fetch_.front();
-    const size_t max_bytes =
+    const size_t max_bytes = std::min(
+        quota_.remaining(),
         std::min(unfinished_work_->config_settings().max_bytes_per_resource(),
                  unfinished_work_->config_settings().max_bytes_total() -
-                     unfinished_work_->total_bytes());
+                     unfinished_work_->total_bytes()));
     VLOG(3) << "Fetching " << resource.first << " " << resource.second;
     pool_.Add(base::MakeUnique<Fetcher>(
         request_context_.get(), resource.first, resource.second,
         base::Bind(&PrecacheFetcher::OnResourceFetchComplete, AsWeakPtr()),
         true /* is_resource_request */, max_bytes));
 
     resources_to_fetch_.pop_front();
   }
 }
 
@@ skipping 16 matching lines @@
     size_t remaining_manifest_urls_to_fetch,
     size_t remaining_resource_urls_to_fetch) {
   RecordCompletionStatistics(*unfinished_work_,
                              remaining_manifest_urls_to_fetch,
                              remaining_resource_urls_to_fetch);
   precache_delegate_->OnDone();
 }
 
 void PrecacheFetcher::StartNextFetch() {
   DCHECK(unfinished_work_->has_config_settings());
-  // If over the precache total size cap, then stop prefetching.
-  if (unfinished_work_->total_bytes() >
-      unfinished_work_->config_settings().max_bytes_total()) {
+
+  // If over the precache total size cap or daily quota, then stop prefetching.
+  if ((unfinished_work_->total_bytes() >
+       unfinished_work_->config_settings().max_bytes_total()) ||
+      quota_.remaining() == 0) {
     size_t pending_manifests_in_pool = 0;
     size_t pending_resources_in_pool = 0;
     for (const auto& element_pair : pool_.elements()) {
       const Fetcher* fetcher = element_pair.first;
       if (fetcher->is_resource_request())
         pending_resources_in_pool++;
       else if (fetcher->url() != config_url_)
         pending_manifests_in_pool++;
     }
     pool_.DeleteAll();
@@ skipping 93 matching lines @@
           manifest.manifest_url, "manifest",
           std::to_string(manifest.manifest_id));
       manifest.manifest_url = net::AppendOrReplaceQueryParameter(
           manifest.manifest_url, "used_resources", manifest.used_url_hash);
       manifest.manifest_url = net::AppendOrReplaceQueryParameter(
           manifest.manifest_url, "unused_resources", manifest.unused_url_hash);
       DCHECK(manifest.manifest_url.is_valid());
     }
   }
   unfinished_work_->set_num_manifest_urls(top_hosts_to_fetch_.size());
+
+  PostTaskAndReplyWithResult(
+      db_task_runner_.get(), FROM_HERE,
+      base::Bind(&RetrieveQuotaInfo, precache_database_),
+      base::Bind(&PrecacheFetcher::OnQuotaInfoRetrieved, AsWeakPtr()));
+}
+
+void PrecacheFetcher::OnQuotaInfoRetrieved(const PrecacheQuota& quota) {
+  quota_ = quota;
+  base::Time time_now = base::Time::Now();
+  if (IsQuotaTimeExpired(quota_, time_now)) {
+    // This is a new day. Update daily quota, that starts today and expires by
+    // end of today.
+    quota_.set_start_time(time_now.LocalMidnight().ToInternalValue());
+    quota_.set_expiry_hours(24);

[sclittle, 2016/09/22 20:02:27]

You might not need to store expiry_hours in the proto - it's always 24 hours,
right?

I think the question here is, if in the future you wanted to change the quota to
e.g. expire every 12 hours, then would you want the old quota in the database to
expire in 12 hours or 24 hours?

If you'd need it to expire in 24 hours, then you should keep expiry_hours in the
proto like you do now. If it'd be OK for the old quota to expire in 12 hours,
then I'd recommend removing expiry_hours for simplicity.

[Raj, 2016/09/22 21:09:10]

Done. I had the expiry time in proto, since quota means a specific start time
and an expiry time.

If expiry time is changed from 24 to 12 hours, the old quota can expire in
either 24 or 12 hours. There is no strict requirement. 

+    quota_.set_remaining(
+        unfinished_work_->config_settings().daily_quota_total());
+    db_task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&PrecacheDatabase::SaveQuota, precache_database_, quota_));
+  }
   StartNextFetch();
 }
 
 ManifestHostInfo::ManifestHostInfo(int64_t manifest_id,
                                    const std::string& hostname,
                                    const std::string& used_url_hash,
                                    const std::string& unused_url_hash)
     : manifest_id(manifest_id),
       hostname(hostname),
       used_url_hash(used_url_hash),
@@ skipping 54 matching lines @@
   // nothing more needs to be done for the resource.
   StartNextFetch();
 }
 
 void PrecacheFetcher::UpdateStats(int64_t response_bytes,
                                   int64_t network_response_bytes) {
   unfinished_work_->set_total_bytes(
       unfinished_work_->total_bytes() + response_bytes);
   unfinished_work_->set_network_bytes(
       unfinished_work_->network_bytes() + network_response_bytes);
+
+  if (!IsQuotaTimeExpired(quota_, base::Time::Now())) {
+    int64_t remaining =
+        static_cast<int64_t>(quota_.remaining()) - network_response_bytes;
+    if (remaining < 0)
+      remaining = 0;
+    quota_.set_remaining(static_cast<uint64_t>(remaining));
+    db_task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&PrecacheDatabase::SaveQuota, precache_database_, quota_));
+  }
 }
 
 }  // namespace precache