Improve IndexedDB's quota support

content/browser/in_process_webkit/indexed_db_dispatcher_host.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/in_process_webkit/indexed_db_dispatcher_host.h"
 
 #include "base/command_line.h"
 #include "base/utf_string_conversions.h"
 #include "content/browser/browser_thread.h"
 #include "content/browser/in_process_webkit/indexed_db_callbacks.h"
@@ skipping 137 matching lines @@
   return cursor_dispatcher_host_->map_.Add(idb_cursor);
 }
 
 int32 IndexedDBDispatcherHost::Add(WebIDBDatabase* idb_database,
                                    const GURL& origin_url) {
   if (!database_dispatcher_host_.get()) {
     delete idb_database;
     return 0;
   }
   int32 idb_database_id = database_dispatcher_host_->map_.Add(idb_database);
-  database_dispatcher_host_->url_map_[idb_database_id] = origin_url;
+  database_dispatcher_host_->database_url_map_[idb_database_id] = origin_url;
   return idb_database_id;
 }
 
 int32 IndexedDBDispatcherHost::Add(WebIDBIndex* idb_index) {
   if (!index_dispatcher_host_.get())  {
     delete idb_index;
     return 0;
   }
   if (!idb_index)
     return 0;
   return index_dispatcher_host_->map_.Add(idb_index);
 }
 
 int32 IndexedDBDispatcherHost::Add(WebIDBObjectStore* idb_object_store) {
   if (!object_store_dispatcher_host_.get()) {
     delete idb_object_store;
     return 0;
   }
   if (!idb_object_store)
     return 0;
   return object_store_dispatcher_host_->map_.Add(idb_object_store);
 }
 
-int32 IndexedDBDispatcherHost::Add(WebIDBTransaction* idb_transaction) {
+int32 IndexedDBDispatcherHost::Add(WebIDBTransaction* idb_transaction,
+                                   const GURL& url) {
   if (!transaction_dispatcher_host_.get()) {
     delete idb_transaction;
     return 0;
   }
   int32 id = transaction_dispatcher_host_->map_.Add(idb_transaction);
   idb_transaction->setCallbacks(new IndexedDBTransactionCallbacks(this, id));
+  transaction_dispatcher_host_->transaction_url_map_[id] = url;
   return id;
 }
 
 void IndexedDBDispatcherHost::OnIDBFactoryOpen(
     const IndexedDBHostMsg_FactoryOpen_Params& params) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   FilePath base_path = webkit_context_->data_path();
   FilePath indexed_db_path;
   if (!base_path.empty()) {
     indexed_db_path = base_path.Append(
         IndexedDBContext::kIndexedDBDirectory);
   }
 
   // TODO(jorlow): This doesn't support file:/// urls properly. We probably need

[michaeln, 2011/07/27 01:31:35]

see DatabaseUtil::GetOriginFromIdentifier

[dgrogan, 2011/07/29 18:14:04]

Thanks.  I'll fix it in another CL.

   //               to add some toString method to WebSecurityOrigin that doesn't
   //               return null for them.
   WebSecurityOrigin origin(
       WebSecurityOrigin::createFromDatabaseIdentifier(params.origin));
   GURL origin_url(origin.toString());
 
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   DCHECK(kDefaultQuota == params.maximum_size);
 
   uint64 quota = kDefaultQuota;
@@ skipping 36 matching lines @@
   GURL url(origin.toString());
 
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   Context()->GetIDBFactory()->deleteDatabase(
       params.name,
       new IndexedDBCallbacks<WebIDBDatabase>(this, params.response_id, url),
       WebSecurityOrigin::createFromDatabaseIdentifier(params.origin), NULL,
       webkit_glue::FilePathToWebString(indexed_db_path));
 }
 
+void IndexedDBDispatcherHost::TransactionComplete(int32 transaction_id) {
+  int bytes_used =
+      transaction_dispatcher_host_->transaction_size_map_[transaction_id];
+  if (bytes_used)
+    Context()->quota_manager_proxy()->NotifyStorageModified(
+        quota::QuotaClient::kIndexedDatabase,
+        transaction_dispatcher_host_->transaction_url_map_[transaction_id],
+        quota::kStorageTypeTemporary,
+        bytes_used);
+  transaction_dispatcher_host_->transaction_size_map_.erase(transaction_id);
+}
+
 //////////////////////////////////////////////////////////////////////
 // Helper templates.
 //
 
 template <typename ObjectType>
 ObjectType* IndexedDBDispatcherHost::GetOrTerminateProcess(
     IDMap<ObjectType, IDMapOwnPointer>* map, int32 return_object_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   ObjectType* return_object = map->Lookup(return_object_id);
   if (!return_object) {
@@ skipping 129 matching lines @@
     WebKit::WebExceptionCode* ec) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   WebIDBDatabase* idb_database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
   if (!idb_database)
     return;
 
   *ec = 0;
   idb_database->setVersion(
       version,
-      new IndexedDBCallbacks<WebIDBTransaction>(parent_, response_id),
+      new IndexedDBCallbacks<WebIDBTransaction>(parent_, response_id,
+          database_url_map_[idb_database_id]),
       *ec);
 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnTransaction(
     int32 idb_database_id,
     const std::vector<string16>& names,
     int32 mode,
     int32 timeout,
     int32* idb_transaction_id,
     WebKit::WebExceptionCode* ec) {
   WebIDBDatabase* database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
   if (!database)
       return;
 
   WebDOMStringList object_stores;
   for (std::vector<string16>::const_iterator it = names.begin();
        it != names.end(); ++it) {
     object_stores.append(*it);
   }
 
   *ec = 0;
   WebIDBTransaction* transaction = database->transaction(
       object_stores, mode, timeout, *ec);
   DCHECK(!transaction != !*ec);
-  *idb_transaction_id = *ec ? 0 : parent_->Add(transaction);
+  *idb_transaction_id =
+      *ec ? 0 : parent_->Add(transaction, database_url_map_[idb_database_id]);
 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnOpen(
     int32 idb_database_id, int32 response_id) {
   WebIDBDatabase* database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
   database->open(new IndexedDBDatabaseCallbacks(parent_, response_id));
 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnClose(
     int32 idb_database_id) {
   WebIDBDatabase* database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
   database->close();
   parent_->Context()->quota_manager_proxy()->NotifyStorageAccessed(
-      quota::QuotaClient::kIndexedDatabase, url_map_[idb_database_id],
+      quota::QuotaClient::kIndexedDatabase, database_url_map_[idb_database_id],
       quota::kStorageTypeTemporary);
-  url_map_.erase(idb_database_id);
+  database_url_map_.erase(idb_database_id);
 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnDestroyed(
     int32 object_id) {
   parent_->DestroyObject(&map_, object_id);
 }
 
 
 //////////////////////////////////////////////////////////////////////
 // IndexedDBDispatcherHost::IndexDispatcherHost
@@ skipping 219 matching lines @@
       &parent_->transaction_dispatcher_host_->map_, transaction_id);
   if (!idb_transaction || !idb_object_store)
     return;
 
   *ec = 0;
   scoped_ptr<WebIDBCallbacks> callbacks(
       new IndexedDBCallbacks<WebSerializedScriptValue>(parent_, response_id));
   idb_object_store->get(key, callbacks.release(), *idb_transaction, *ec);
 }
 
+static void AbortTransaction(WebIDBTransaction* idb_transaction) {
+  idb_transaction->abort();

[michaeln, 2011/07/27 01:31:35]

could idb_transaction have been deleted prior to this function call... ditto
question about the function below?

+}
+
+static void DidCompleteTaskEvents(WebIDBTransaction* idb_transaction) {
+  idb_transaction->didCompleteTaskEvents();
+}
+
+class IndexedDBGetUsageAndQuotaCallback :
+    public quota::QuotaManager::GetUsageAndQuotaCallback {
+ public:
+  IndexedDBGetUsageAndQuotaCallback(int64 size,
+                                    WebIDBTransaction* idb_transaction)
+      : size_(size),
+      idb_transaction_(idb_transaction) {

[michaeln, 2011/07/27 01:31:35]

nit: indent is off

+  }
+
+  void Run(quota::QuotaStatusCode status, int64 usage, int64 quota) {
+    if (size_ > (quota - usage)) {
+      BrowserThread::PostTask(BrowserThread::WEBKIT, FROM_HERE,
+          NewRunnableFunction(&AbortTransaction, idb_transaction_));
+    } else {
+      BrowserThread::PostTask(BrowserThread::WEBKIT, FROM_HERE,
+          NewRunnableFunction(&DidCompleteTaskEvents, idb_transaction_));
+    }
+  }
+
+  virtual void RunWithParams(
+        const Tuple3<quota::QuotaStatusCode, int64, int64>& params) {
+    Run(params.a, params.b, params.c);
+  }
+
+ private:
+  int64 size_;
+  WebIDBTransaction* idb_transaction_;
+};
+
+static void RequestQuota(
+    scoped_refptr<quota::QuotaManagerProxy> proxy, const GURL& gurl,
+    IndexedDBGetUsageAndQuotaCallback* idb_quota_callback) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  proxy->quota_manager()->GetUsageAndQuota(
+      gurl,
+      quota::kStorageTypeTemporary,
+      idb_quota_callback);
+}
+
 void IndexedDBDispatcherHost::ObjectStoreDispatcherHost::OnPut(
     const IndexedDBHostMsg_ObjectStorePut_Params& params,
     WebKit::WebExceptionCode* ec) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   WebIDBObjectStore* idb_object_store = parent_->GetOrTerminateProcess(
       &map_, params.idb_object_store_id);
   WebIDBTransaction* idb_transaction = parent_->GetOrTerminateProcess(
       &parent_->transaction_dispatcher_host_->map_, params.transaction_id);
   if (!idb_transaction || !idb_object_store)
     return;
 
   *ec = 0;
   scoped_ptr<WebIDBCallbacks> callbacks(
       new IndexedDBCallbacks<WebIDBKey>(parent_, params.response_id));
+
   idb_object_store->put(params.serialized_value, params.key, params.put_mode,
                         callbacks.release(), *idb_transaction, *ec);
+  if (*ec)
+    return;
+  // TODO(dgrogan): Count key size too.
+  // TODO(dgrogan): Figure out how accurate this is in terms of disk space used.

[michaeln, 2011/07/27 01:31:35]

Eavesdropping on OnPut/OnDelete/OnClear/OnIndex/OnDeleteIndex look like a
reasonable way to determine if something has changed in the transaction, but i'm
very skeptical of this technique to determine by how much its change by...
presumably puts can overwrite existing values?

I think it would be good to think thru some other approaches to measuring how
much.

[dgrogan, 2011/07/27 22:56:01]

Your concerns are valid.  The issue is that LevelDB only appends to files in
direct response to API calls.  It records a note to delete values and overwrite
values.  The original values are left intact until a compaction, which happens
according to LevelDB's internal heuristics.  I erred mostly on the side of
overcounting by not counting overwrites, deletes, or deleting an entire object
store.  I say "mostly" because the disk space used to temporarily store the
delete/overwrite markers is not counted against the quota.

My naive theory is that an application can trigger a call to
IndexedDBQuotaClient.GetUsage() via the javascript quota API, which checks the
size of the leveldb directory on disk, and future reported deltas would be
counted against that.  If that's not accurate, the worst case is that the
application would think it has run out of quota before it actually does, but
would get an accurate count when the browser is restarted.
You're right, we need to get an approximate number of bytes of file system space
used from leveldb, which it mostly supports.  I fell back to the current
IPC-sniffing chrome-side-only implementation as the deadline for m14 drew near
:(

[dgrogan, 2011/07/29 18:14:04]

So, I realized that's not how the quota manager works, it just queries indexeddb
for size on disk at browser startup.
This seems to be the situation.  I think it's my best option for m14.  WDYT?

+  int64 size = UTF16ToUTF8(params.serialized_value.data()).size();
+  WebIDBTransactionIDToSizeMap* map =
+      &parent_->transaction_dispatcher_host_->transaction_size_map_;
+  (*map)[params.transaction_id] += size;
 }
 
 void IndexedDBDispatcherHost::ObjectStoreDispatcherHost::OnDelete(
     int idb_object_store_id,
     int32 response_id,
     const IndexedDBKey& key,
     int32 transaction_id,
     WebKit::WebExceptionCode* ec) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   WebIDBObjectStore* idb_object_store = parent_->GetOrTerminateProcess(
@@ skipping 297 matching lines @@
 
 void IndexedDBDispatcherHost::TransactionDispatcherHost::OnObjectStore(
     int32 transaction_id, const string16& name, int32* object_store_id,
     WebKit::WebExceptionCode* ec) {
   WebIDBTransaction* idb_transaction = parent_->GetOrTerminateProcess(
       &map_, transaction_id);
   if (!idb_transaction)
     return;
 
   *ec = 0;
   WebIDBObjectStore* object_store = idb_transaction->objectStore(name, *ec);

[michaeln, 2011/07/27 01:31:35]

this looks like a mutation too

   *object_store_id = object_store ? parent_->Add(object_store) : 0;
 }
 
 void IndexedDBDispatcherHost::
     TransactionDispatcherHost::OnDidCompleteTaskEvents(int transaction_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   WebIDBTransaction* idb_transaction = parent_->GetOrTerminateProcess(
       &map_, transaction_id);
   if (!idb_transaction)
     return;
 
-  idb_transaction->didCompleteTaskEvents();
+  if (transaction_size_map_.find(transaction_id) ==
+      transaction_size_map_.end()) {

[michaeln, 2011/07/27 01:31:35]

i see... no Puts(), so no size map entry, so nothing todo

+    idb_transaction->didCompleteTaskEvents();
+    return;
+  }
+  // Check if there is available quota.
+  IndexedDBGetUsageAndQuotaCallback* callback =
+      new IndexedDBGetUsageAndQuotaCallback(
+          transaction_size_map_[transaction_id], idb_transaction);
+  BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
+      NewRunnableFunction(&RequestQuota,
+                        scoped_refptr<quota::QuotaManagerProxy>(
+                            parent_->Context()->quota_manager_proxy()),
+                        transaction_url_map_[transaction_id],
+                        callback));
+
 }
 
 void IndexedDBDispatcherHost::TransactionDispatcherHost::OnDestroyed(
     int32 object_id) {
   parent_->DestroyObject(&map_, object_id);

[michaeln, 2011/07/27 01:31:35]

this might be a good place to erase from the transaction_size_map_

 }