AppCache + Quota integration

webkit/appcache/appcache_storage_impl.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "webkit/appcache/appcache_storage_impl.h"
 
+#include <set>
+
 #include "app/sql/connection.h"
 #include "app/sql/transaction.h"
 #include "base/file_util.h"
 #include "base/logging.h"
 #include "base/message_loop.h"
 #include "base/stl_util-inl.h"
 #include "base/string_util.h"
 #include "net/base/cache_type.h"
 #include "net/base/net_errors.h"
 #include "webkit/appcache/appcache.h"
 #include "webkit/appcache/appcache_database.h"
 #include "webkit/appcache/appcache_entry.h"
 #include "webkit/appcache/appcache_group.h"
 #include "webkit/appcache/appcache_histograms.h"
 #include "webkit/appcache/appcache_policy.h"
+#include "webkit/appcache/appcache_quota_client.h"
 #include "webkit/appcache/appcache_response.h"
 #include "webkit/appcache/appcache_service.h"
 #include "webkit/appcache/appcache_thread.h"
+#include "webkit/quota/quota_client.h"
+#include "webkit/quota/quota_manager.h"
 #include "webkit/quota/special_storage_policy.h"
 
 namespace {
 // Helper with no return value for use with NewRunnableFunction.
 void DeleteDirectory(const FilePath& path) {
   file_util::Delete(path, true);
 }
 }
 
 namespace appcache {
 
+// Hard coded default when not using quota management.
+static const int kDefaultQuota = 5 * 1024 * 1024;
+
 static const int kMaxDiskCacheSize = 250 * 1024 * 1024;
 static const int kMaxMemDiskCacheSize = 10 * 1024 * 1024;
 static const FilePath::CharType kDiskCacheDirectoryName[] =
     FILE_PATH_LITERAL("Cache");
 
+
 // DatabaseTask -----------------------------------------
 
 class AppCacheStorageImpl::DatabaseTask
     : public base::RefCountedThreadSafe<DatabaseTask> {
  public:
   explicit DatabaseTask(AppCacheStorageImpl* storage)
       : storage_(storage), database_(storage->database_) {}
 
   virtual ~DatabaseTask() {}
 
@@ skipping 88 matching lines @@
         last_cache_id_(0), last_response_id_(0),
         last_deletable_response_rowid_(0) {}
 
   virtual void Run();
   virtual void RunCompleted();
 
   int64 last_group_id_;
   int64 last_cache_id_;
   int64 last_response_id_;
   int64 last_deletable_response_rowid_;
-  std::set<GURL> origins_with_groups_;
+  std::map<GURL, int64> usage_map_;
 };
 
 void AppCacheStorageImpl::InitTask::Run() {
   database_->FindLastStorageIds(
       &last_group_id_, &last_cache_id_, &last_response_id_,
       &last_deletable_response_rowid_);
-  database_->FindOriginsWithGroups(&origins_with_groups_);
+  database_->GetAllOriginUsage(&usage_map_);
 }
 
 void AppCacheStorageImpl::InitTask::RunCompleted() {
   storage_->last_group_id_ = last_group_id_;
   storage_->last_cache_id_ = last_cache_id_;
   storage_->last_response_id_ = last_response_id_;
   storage_->last_deletable_response_rowid_ = last_deletable_response_rowid_;
 
   if (!storage_->is_disabled()) {
-    storage_->origins_with_groups_.swap(origins_with_groups_);
-
+    storage_->usage_map_.swap(usage_map_);
     const int kDelayMillis = 5 * 60 * 1000;  // Five minutes.
     MessageLoop::current()->PostDelayedTask(FROM_HERE,
       storage_->method_factory_.NewRunnableMethod(
           &AppCacheStorageImpl::DelayedStartDeletingUnusedResponses),
       kDelayMillis);
   }
+
+  if (storage_->service()->quota_client())
+    storage_->service()->quota_client()->NotifyAppCacheReady();
 }
 
 // CloseConnectionTask -------
 
 class AppCacheStorageImpl::CloseConnectionTask : public DatabaseTask {
  public:
   explicit CloseConnectionTask(AppCacheStorageImpl* storage)
       : DatabaseTask(storage) {}
 
   virtual void Run() { database_->CloseConnection(); }
@@ skipping 111 matching lines @@
   // We have to update foriegn entries if MarkEntryAsForeignTasks
   // are in flight.
   std::vector<GURL> urls;
   storage_->GetPendingForeignMarkingsForCache(cache->get()->cache_id(), &urls);
   for (std::vector<GURL>::iterator iter = urls.begin();
        iter != urls.end(); ++iter) {
     DCHECK(cache->get()->GetEntry(*iter));
     cache->get()->GetEntry(*iter)->add_types(AppCacheEntry::FOREIGN);
   }
 
+  storage_->NotifyStorageAccessed(group_record_.origin);
+
   // TODO(michaeln): Maybe verify that the responses we expect to exist
   // do actually exist in the disk_cache (and if not then what?)
 }
 
 // CacheLoadTask -------
 
 class AppCacheStorageImpl::CacheLoadTask : public StoreOrLoadTask {
  public:
   CacheLoadTask(int64 cache_id, AppCacheStorageImpl* storage)
       : StoreOrLoadTask(storage), cache_id_(cache_id),
@@ skipping 73 matching lines @@
   FOR_EACH_DELEGATE(delegates_, OnGroupLoaded(group, manifest_url_));
 }
 
 // StoreGroupAndCacheTask -------
 
 class AppCacheStorageImpl::StoreGroupAndCacheTask : public StoreOrLoadTask {
  public:
   StoreGroupAndCacheTask(AppCacheStorageImpl* storage, AppCacheGroup* group,
                          AppCache* newest_cache);
 
+  void GetQuotaThenSchedule();
+  void OnQuotaCallback(
+      quota::QuotaStatusCode status, int64 usage, int64 quota);
+
   virtual void Run();
   virtual void RunCompleted();
   virtual void CancelCompletion();
 
   scoped_refptr<AppCacheGroup> group_;
   scoped_refptr<AppCache> cache_;
   bool success_;
   bool would_exceed_quota_;
-  int64 quota_override_;
+  int64 space_available_;
+  int64 new_origin_usage_;
   std::vector<int64> newly_deletable_response_ids_;
 };
 
 AppCacheStorageImpl::StoreGroupAndCacheTask::StoreGroupAndCacheTask(
     AppCacheStorageImpl* storage, AppCacheGroup* group, AppCache* newest_cache)
     : StoreOrLoadTask(storage), group_(group), cache_(newest_cache),
       success_(false), would_exceed_quota_(false),
-      quota_override_(-1) {
+      space_available_(-1), new_origin_usage_(-1) {
   group_record_.group_id = group->group_id();
   group_record_.manifest_url = group->manifest_url();
   group_record_.origin = group_record_.manifest_url.GetOrigin();
   newest_cache->ToDatabaseRecords(
       group,
       &cache_record_, &entry_records_, &fallback_namespace_records_,
       &online_whitelist_records_);
+}
 
-  if (storage->service()->special_storage_policy() &&
-      storage->service()->special_storage_policy()->IsStorageUnlimited(
-          group_record_.origin)) {
-    quota_override_ = kint64max;
+void AppCacheStorageImpl::StoreGroupAndCacheTask::GetQuotaThenSchedule() {
+  quota::QuotaManager* quota_manager = NULL;
+  if (storage_->service()->quota_manager_proxy()) {
+    quota_manager =
+        storage_->service()->quota_manager_proxy()->quota_manager();
   }
+
+  if (!quota_manager) {
+    if (storage_->service()->special_storage_policy() &&
+        storage_->service()->special_storage_policy()->IsStorageUnlimited(
+            group_record_.origin))
+      space_available_ = kint64max;
+    Schedule();
+    return;
+  }
+
+  // We have to ask the quota manager for the value.
+  AddRef(); // balanced in the OnQuotaCallback
+  storage_->pending_quota_queries_.insert(this);
+  quota_manager->GetUsageAndQuota(
+      group_record_.origin, quota::kStorageTypeTemporary,
+      NewCallback(this, &StoreGroupAndCacheTask::OnQuotaCallback));
+}
+
+void AppCacheStorageImpl::StoreGroupAndCacheTask::OnQuotaCallback(
+    quota::QuotaStatusCode status, int64 usage, int64 quota) {
+  // TODO(michaeln): what do we do if status is not OK, fail the store op?

[michaeln, 2011/06/08 18:09:54]

Took care of this TODO by setting space_available_ to 0 when status != OK.

+  DCHECK_EQ(quota::kQuotaStatusOk, status);
+  space_available_ = std::max(static_cast<int64>(0), quota - usage);
+  if (storage_) {
+    storage_->pending_quota_queries_.erase(this);
+    Schedule();
+  }
+  Release(); // balanced in GetQuotaThenSchedule
 }
 
 void AppCacheStorageImpl::StoreGroupAndCacheTask::Run() {
   DCHECK(!success_);
   sql::Connection* connection = database_->db_connection();
   if (!connection)
     return;
 
   sql::Transaction transaction(connection);
   if (!transaction.Begin())
     return;
 
+  int64 old_origin_usage = database_->GetOriginUsage(group_record_.origin);
+
   AppCacheDatabase::GroupRecord existing_group;
   success_ = database_->FindGroup(group_record_.group_id, &existing_group);
   if (!success_) {
     group_record_.creation_time = base::Time::Now();
     group_record_.last_access_time = base::Time::Now();
     success_ = database_->InsertGroup(&group_record_);
   } else {
     DCHECK(group_record_.group_id == existing_group.group_id);
     DCHECK(group_record_.manifest_url == existing_group.manifest_url);
     DCHECK(group_record_.origin == existing_group.origin);
@@ skipping 37 matching lines @@
   success_ =
       success_ &&
       database_->InsertCache(&cache_record_) &&
       database_->InsertEntryRecords(entry_records_) &&
       database_->InsertFallbackNameSpaceRecords(fallback_namespace_records_)&&
       database_->InsertOnlineWhiteListRecords(online_whitelist_records_);
 
   if (!success_)
     return;
 
-  int64 quota = (quota_override_ >= 0) ?
-      quota_override_ :
-      database_->GetOriginQuota(group_record_.origin);
+  new_origin_usage_ = database_->GetOriginUsage(group_record_.origin);
 
-  if (database_->GetOriginUsage(group_record_.origin) > quota) {
+  // Only check quota when the new usage exceeds the old usage.
+  if (new_origin_usage_ <= old_origin_usage) {
+    success_ = transaction.Commit();
+    return;
+  }
+
+  // Use a simple hard-coded value when not using quota managment.
+  if (space_available_ == -1) {
+    if (new_origin_usage_ > kDefaultQuota) {
+      would_exceed_quota_ = true;
+      success_ = false;
+      return;
+    }
+    success_ = transaction.Commit();
+    return;
+  }
+
+  // Check limits based on the space availbable given to us via the
+  // quota system.
+  int64 delta = new_origin_usage_ - old_origin_usage;
+  if (delta > space_available_) {
     would_exceed_quota_ = true;
     success_ = false;
     return;
   }
 
   success_ = transaction.Commit();
 }
 
 void AppCacheStorageImpl::StoreGroupAndCacheTask::RunCompleted() {
   if (success_) {
-    storage_->origins_with_groups_.insert(group_->manifest_url().GetOrigin());
+    storage_->UpdateUsageMapAndNotify(
+        group_->manifest_url().GetOrigin(), new_origin_usage_);
     if (cache_ != group_->newest_complete_cache()) {
       cache_->set_complete(true);
       group_->AddCache(cache_);
     }
     if (group_->creation_time().is_null())
       group_->set_creation_time(group_record_.creation_time);
     group_->AddNewlyDeletableResponseIds(&newly_deletable_response_ids_);
   }
   FOR_EACH_DELEGATE(delegates_,
                     OnGroupAndNewestCacheStored(group_, cache_, success_,
@@ skipping 307 matching lines @@
 class AppCacheStorageImpl::MakeGroupObsoleteTask : public DatabaseTask {
  public:
   MakeGroupObsoleteTask(AppCacheStorageImpl* storage, AppCacheGroup* group);
 
   virtual void Run();
   virtual void RunCompleted();
   virtual void CancelCompletion();
 
   scoped_refptr<AppCacheGroup> group_;
   int64 group_id_;
+  GURL origin_;
   bool success_;
-  std::set<GURL> origins_with_groups_;
+  int64 new_origin_usage_;
   std::vector<int64> newly_deletable_response_ids_;
 };
 
 AppCacheStorageImpl::MakeGroupObsoleteTask::MakeGroupObsoleteTask(
     AppCacheStorageImpl* storage, AppCacheGroup* group)
     : DatabaseTask(storage), group_(group), group_id_(group->group_id()),
-      success_(false) {
+      origin_(group->manifest_url().GetOrigin()),
+      success_(false), new_origin_usage_(-1) {
 }
 
 void AppCacheStorageImpl::MakeGroupObsoleteTask::Run() {
   DCHECK(!success_);
   sql::Connection* connection = database_->db_connection();
   if (!connection)
     return;
 
   sql::Transaction transaction(connection);
   if (!transaction.Begin())
     return;
 
   AppCacheDatabase::GroupRecord group_record;
   if (!database_->FindGroup(group_id_, &group_record)) {
     // This group doesn't exists in the database, nothing todo here.
+    new_origin_usage_ = database_->GetOriginUsage(origin_);
     success_ = true;
     return;
   }
 
+  DCHECK_EQ(group_record.origin, origin_);
+
   AppCacheDatabase::CacheRecord cache_record;
   if (database_->FindCacheForGroup(group_id_, &cache_record)) {
     database_->FindResponseIdsForCacheAsVector(cache_record.cache_id,
                                                &newly_deletable_response_ids_);
     success_ =
         database_->DeleteGroup(group_id_) &&
         database_->DeleteCache(cache_record.cache_id) &&
         database_->DeleteEntriesForCache(cache_record.cache_id) &&
         database_->DeleteFallbackNameSpacesForCache(cache_record.cache_id) &&
         database_->DeleteOnlineWhiteListForCache(cache_record.cache_id) &&
         database_->InsertDeletableResponseIds(newly_deletable_response_ids_);
   } else {
     NOTREACHED() << "A existing group without a cache is unexpected";
     success_ = database_->DeleteGroup(group_id_);
   }
 
-  success_ = success_ &&
-             database_->FindOriginsWithGroups(&origins_with_groups_) &&
-             transaction.Commit();
+  new_origin_usage_ = database_->GetOriginUsage(origin_);
+  success_ = success_ && transaction.Commit();
 }
 
 void AppCacheStorageImpl::MakeGroupObsoleteTask::RunCompleted() {
   if (success_) {
     group_->set_obsolete(true);
     if (!storage_->is_disabled()) {
-      storage_->origins_with_groups_.swap(origins_with_groups_);
+      storage_->UpdateUsageMapAndNotify(origin_, new_origin_usage_);
       group_->AddNewlyDeletableResponseIds(&newly_deletable_response_ids_);
 
       // Also remove from the working set, caches for an 'obsolete' group
       // may linger in use, but the group itself cannot be looked up by
       // 'manifest_url' in the working set any longer.
       storage_->working_set()->RemoveGroup(group_);
     }
   }
   FOR_EACH_DELEGATE(delegates_, OnGroupMadeObsolete(group_, success_));
   group_ = NULL;
@@ skipping 59 matching lines @@
 void AppCacheStorageImpl::DeleteDeletableResponseIdsTask::Run() {
   database_->DeleteDeletableResponseIds(response_ids_);
 }
 
 // UpdateGroupLastAccessTimeTask -------
 
 class AppCacheStorageImpl::UpdateGroupLastAccessTimeTask
     : public DatabaseTask {
  public:
   UpdateGroupLastAccessTimeTask(
-      AppCacheStorageImpl* storage, int64 group_id, base::Time time)
-      : DatabaseTask(storage), group_id_(group_id), last_access_time_(time) {}
+      AppCacheStorageImpl* storage, AppCacheGroup* group, base::Time time)
+      : DatabaseTask(storage), group_id_(group->group_id()),
+        last_access_time_(time) {
+    storage->NotifyStorageAccessed(group->manifest_url().GetOrigin());
+  }
 
   virtual void Run();
 
   int64 group_id_;
   base::Time last_access_time_;
 };
 
 void AppCacheStorageImpl::UpdateGroupLastAccessTimeTask::Run() {
   database_->UpdateGroupLastAccessTime(group_id_, last_access_time_);
 }
@@ skipping 10 matching lines @@
           this, &AppCacheStorageImpl::OnDeletedOneResponse)),
       ALLOW_THIS_IN_INITIALIZER_LIST(init_callback_(
           this, &AppCacheStorageImpl::OnDiskCacheInitialized)),
       database_(NULL), is_disabled_(false),
       ALLOW_THIS_IN_INITIALIZER_LIST(method_factory_(this)) {
 }
 
 AppCacheStorageImpl::~AppCacheStorageImpl() {
   STLDeleteElements(&pending_simple_tasks_);
 
+  std::for_each(pending_quota_queries_.begin(),
+                pending_quota_queries_.end(),
+                std::mem_fun(&DatabaseTask::CancelCompletion));
   std::for_each(scheduled_database_tasks_.begin(),
                 scheduled_database_tasks_.end(),
                 std::mem_fun(&DatabaseTask::CancelCompletion));
 
   if (database_)
     AppCacheThread::DeleteSoon(AppCacheThread::db(), FROM_HERE, database_);
 }
 
 void AppCacheStorageImpl::Initialize(const FilePath& cache_directory,
                                      base::MessageLoopProxy* cache_thread) {
   cache_directory_ = cache_directory;
   cache_thread_ = cache_thread;
   is_incognito_ = cache_directory_.empty();
 
   FilePath db_file_path;
   if (!is_incognito_)
     db_file_path = cache_directory_.Append(kAppCacheDatabaseName);
   database_ = new AppCacheDatabase(db_file_path);
 
   scoped_refptr<InitTask> task(new InitTask(this));
   task->Schedule();
 }
 
 void AppCacheStorageImpl::Disable() {
   if (is_disabled_)
     return;
   VLOG(1) << "Disabling appcache storage.";
   is_disabled_ = true;
-  origins_with_groups_.clear();
+  usage_map_.clear();  // TODO(michaeln): What to tell the QuotaManager?

[michaeln, 2011/06/08 18:09:54]

Took are of this TODO with ClearUsageMapAndNotify(), we'll send notifcations
that look like all appcache data has been deleted.

   working_set()->Disable();
   if (disk_cache_.get())
     disk_cache_->Disable();
   scoped_refptr<DisableDatabaseTask> task(new DisableDatabaseTask(this));
   task->Schedule();
 }
 
 void AppCacheStorageImpl::GetAllInfo(Delegate* delegate) {
   DCHECK(delegate);
   scoped_refptr<GetAllInfoTask> task(new GetAllInfoTask(this));
   task->AddDelegate(GetOrCreateDelegateReference(delegate));
   task->Schedule();
 }
 
 void AppCacheStorageImpl::LoadCache(int64 id, Delegate* delegate) {
   DCHECK(delegate);
   if (is_disabled_) {
     delegate->OnCacheLoaded(NULL, id);
     return;
   }
 
   AppCache* cache = working_set_.GetCache(id);
   if (cache) {
     delegate->OnCacheLoaded(cache, id);
     if (cache->owning_group()) {
       scoped_refptr<DatabaseTask> update_task(
           new UpdateGroupLastAccessTimeTask(
-              this, cache->owning_group()->group_id(), base::Time::Now()));
+              this, cache->owning_group(), base::Time::Now()));
       update_task->Schedule();
     }
     return;
   }
   scoped_refptr<CacheLoadTask> task(GetPendingCacheLoadTask(id));
   if (task) {
     task->AddDelegate(GetOrCreateDelegateReference(delegate));
     return;
   }
   task = new CacheLoadTask(id, this);
   task->AddDelegate(GetOrCreateDelegateReference(delegate));
   task->Schedule();
   pending_cache_loads_[id] = task;
 }
 
 void AppCacheStorageImpl::LoadOrCreateGroup(
     const GURL& manifest_url, Delegate* delegate) {
   DCHECK(delegate);
   if (is_disabled_) {
     delegate->OnGroupLoaded(NULL, manifest_url);
     return;
   }
 
   AppCacheGroup* group = working_set_.GetGroup(manifest_url);
   if (group) {
     delegate->OnGroupLoaded(group, manifest_url);
     scoped_refptr<DatabaseTask> update_task(
         new UpdateGroupLastAccessTimeTask(
-            this, group->group_id(), base::Time::Now()));
+            this, group, base::Time::Now()));
     update_task->Schedule();
     return;
   }
 
   scoped_refptr<GroupLoadTask> task(GetPendingGroupLoadTask(manifest_url));
   if (task) {
     task->AddDelegate(GetOrCreateDelegateReference(delegate));
     return;
   }
 
-  if (origins_with_groups_.find(manifest_url.GetOrigin()) ==
-      origins_with_groups_.end()) {
+  if (usage_map_.find(manifest_url.GetOrigin()) == usage_map_.end()) {
     // No need to query the database, return a new group immediately.
     scoped_refptr<AppCacheGroup> group(new AppCacheGroup(
         service_, manifest_url, NewGroupId()));
     delegate->OnGroupLoaded(group, manifest_url);
     return;
   }
 
   task = new GroupLoadTask(manifest_url, this);
   task->AddDelegate(GetOrCreateDelegateReference(delegate));
   task->Schedule();
   pending_group_loads_[manifest_url] = task.get();
 }
 
 void AppCacheStorageImpl::StoreGroupAndNewestCache(
     AppCacheGroup* group, AppCache* newest_cache, Delegate* delegate) {
   // TODO(michaeln): distinguish between a simple update of an existing
   // cache that just adds new master entry(s), and the insertion of a
   // whole new cache. The StoreGroupAndCacheTask as written will handle
   // the simple update case in a very heavy weight way (delete all and
   // the reinsert all over again).
   DCHECK(group && delegate && newest_cache);
   scoped_refptr<StoreGroupAndCacheTask> task(
       new StoreGroupAndCacheTask(this, group, newest_cache));
   task->AddDelegate(GetOrCreateDelegateReference(delegate));
-  task->Schedule();
+  task->GetQuotaThenSchedule();
 }
 
 void AppCacheStorageImpl::FindResponseForMainRequest(
     const GURL& url, const GURL& preferred_manifest_url,
     Delegate* delegate) {
   DCHECK(delegate);
 
   const GURL* url_ptr = &url;
   GURL url_no_ref;
   if (url.has_ref()) {
@@ skipping 23 matching lines @@
               groups_in_use->begin();
            it != groups_in_use->end(); ++it) {
         if (FindResponseForMainRequestInGroup(
                 it->second, *url_ptr, delegate)) {
           return;
         }
       }
     }
   }
 
-  if (IsInitTaskComplete() &&
-      origins_with_groups_.find(origin) == origins_with_groups_.end()) {
+  if (IsInitTaskComplete() &&  usage_map_.find(origin) == usage_map_.end()) {
     // No need to query the database, return async'ly but without going thru
     // the DB thread.
     scoped_refptr<AppCacheGroup> no_group;
     scoped_refptr<AppCache> no_cache;
     ScheduleSimpleTask(method_factory_.NewRunnableMethod(
         &AppCacheStorageImpl::DeliverShortCircuitedFindMainResponse,
         url, AppCacheEntry(), no_group, no_cache,
         make_scoped_refptr(GetOrCreateDelegateReference(delegate))));
     return;
   }
@@ skipping 306 matching lines @@
     Disable();
     if (!is_incognito_) {
       VLOG(1) << "Deleting existing appcache data and starting over.";
       AppCacheThread::PostTask(AppCacheThread::db(), FROM_HERE,
           NewRunnableFunction(DeleteDirectory, cache_directory_));
     }
   }
 }
 
 }  // namespace appcache