Improve IndexedDB's quota support

content/browser/in_process_webkit/indexed_db_dispatcher_host.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/in_process_webkit/indexed_db_dispatcher_host.h"
 
 #include "base/command_line.h"
 #include "base/utf_string_conversions.h"
 #include "content/browser/browser_thread.h"
 #include "content/browser/in_process_webkit/indexed_db_callbacks.h"
@@ skipping 10 matching lines @@
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebIDBDatabase.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebIDBDatabaseError.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebIDBKeyRange.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebIDBIndex.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebIDBFactory.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebIDBObjectStore.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebIDBTransaction.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebSecurityOrigin.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebVector.h"
 #include "webkit/glue/webkit_glue.h"
-#include "webkit/quota/quota_manager.h"
 
 using WebKit::WebDOMStringList;
 using WebKit::WebExceptionCode;
 using WebKit::WebIDBCallbacks;
 using WebKit::WebIDBCursor;
 using WebKit::WebIDBDatabase;
 using WebKit::WebIDBDatabaseError;
 using WebKit::WebIDBIndex;
 using WebKit::WebIDBKey;
 using WebKit::WebIDBKeyRange;
@@ skipping 48 matching lines @@
     ResetDispatcherHosts();
 }
 
 void IndexedDBDispatcherHost::ResetDispatcherHosts() {
   // It is important that the various *_dispatcher_host_ members are reset
   // on the WebKit thread, since there might be incoming messages on that
   // thread, and we must not reset the dispatcher hosts until after those
   // messages are processed.
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT) ||
          CommandLine::ForCurrentProcess()->HasSwitch(switches::kSingleProcess));
 

[michaeln, 2011/08/03 03:40:40]

On a child process crash the call sequence will not go thru the graceful
OnClosed/OnDestroyed code paths. If there are context->NewConnection() calls
that have not yet been balanced by a context->ConnectionClosed() call,
IndexedDBContext's bookeeping will get confused. The connection_count_ for those
origins will never drop to zero.

   database_dispatcher_host_.reset();
   index_dispatcher_host_.reset();
   object_store_dispatcher_host_.reset();
   cursor_dispatcher_host_.reset();
   transaction_dispatcher_host_.reset();
 }
 
 void IndexedDBDispatcherHost::OverrideThreadForMessage(
     const IPC::Message& message,
     BrowserThread::ID* thread) {
@@ skipping 37 matching lines @@
   return cursor_dispatcher_host_->map_.Add(idb_cursor);
 }
 
 int32 IndexedDBDispatcherHost::Add(WebIDBDatabase* idb_database,
                                    const GURL& origin_url) {
   if (!database_dispatcher_host_.get()) {
     delete idb_database;
     return 0;
   }
   int32 idb_database_id = database_dispatcher_host_->map_.Add(idb_database);
-  database_dispatcher_host_->url_map_[idb_database_id] = origin_url;
+  Context()->NewConnection(origin_url);
+  database_dispatcher_host_->database_url_map_[idb_database_id] = origin_url;
   return idb_database_id;
 }
 
 int32 IndexedDBDispatcherHost::Add(WebIDBIndex* idb_index) {
   if (!index_dispatcher_host_.get())  {
     delete idb_index;
     return 0;
   }
   if (!idb_index)
     return 0;
   return index_dispatcher_host_->map_.Add(idb_index);
 }
 
 int32 IndexedDBDispatcherHost::Add(WebIDBObjectStore* idb_object_store) {
   if (!object_store_dispatcher_host_.get()) {
     delete idb_object_store;
     return 0;
   }
   if (!idb_object_store)
     return 0;
   return object_store_dispatcher_host_->map_.Add(idb_object_store);
 }
 
-int32 IndexedDBDispatcherHost::Add(WebIDBTransaction* idb_transaction) {
+int32 IndexedDBDispatcherHost::Add(WebIDBTransaction* idb_transaction,
+                                   const GURL& url) {
   if (!transaction_dispatcher_host_.get()) {
     delete idb_transaction;
     return 0;
   }
   int32 id = transaction_dispatcher_host_->map_.Add(idb_transaction);
   idb_transaction->setCallbacks(new IndexedDBTransactionCallbacks(this, id));
+  transaction_dispatcher_host_->transaction_url_map_[id] = url;
   return id;
 }
 
 void IndexedDBDispatcherHost::OnIDBFactoryOpen(
     const IndexedDBHostMsg_FactoryOpen_Params& params) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   FilePath base_path = webkit_context_->data_path();
   FilePath indexed_db_path;
   if (!base_path.empty()) {
     indexed_db_path = base_path.Append(
         IndexedDBContext::kIndexedDBDirectory);
   }
 
   // TODO(jorlow): This doesn't support file:/// urls properly. We probably need
   //               to add some toString method to WebSecurityOrigin that doesn't
   //               return null for them.
   WebSecurityOrigin origin(
       WebSecurityOrigin::createFromDatabaseIdentifier(params.origin));
   GURL origin_url(origin.toString());
 
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   DCHECK(kDefaultQuota == params.maximum_size);
 
   uint64 quota = kDefaultQuota;

[michaeln, 2011/08/03 03:40:40]

Is this old quota code needed anymore? Is the value of this param used by
GetIDBFactory()->open(... quota) anymore?

[dgrogan, 2011/08/04 19:47:50]

Not by default, only if --indexeddb-use-sqlite is specified.  In that case the
user knows what they're doing and quota can probably be left out.

So, this should be removed, but maybe in a future patch.

   if (Context()->IsUnlimitedStorageGranted(origin_url) ||
       CommandLine::ForCurrentProcess()->HasSwitch(
           switches::kUnlimitedQuotaForIndexedDB)) {
       // TODO(jorlow): For the IsUnlimitedStorageGranted case, we need some
       //       way to revoke it.
       // TODO(jorlow): Use kint64max once we think we can scale over 1GB.
       quota = 1024 * 1024 * 1024; // 1GB. More or less "unlimited".
   }
 
   WebKit::WebIDBFactory::BackingStoreType backingStoreType =
       WebKit::WebIDBFactory::LevelDBBackingStore;
 
   if (CommandLine::ForCurrentProcess()->HasSwitch(
       switches::kSQLiteIndexedDatabase)) {
     backingStoreType = WebKit::WebIDBFactory::SQLiteBackingStore;
   }
 
+  // TODO(dgrogan): Don't let a non-existing database be opened (and therefore
+  // created) if this origin is already over quota.
   Context()->GetIDBFactory()->open(
       params.name,
       new IndexedDBCallbacks<WebIDBDatabase>(this, params.response_id,
                                              origin_url),
       origin, NULL, webkit_glue::FilePathToWebString(indexed_db_path), quota,
       backingStoreType);
 }
 
 void IndexedDBDispatcherHost::OnIDBFactoryDeleteDatabase(
     const IndexedDBHostMsg_FactoryDeleteDatabase_Params& params) {
   FilePath base_path = webkit_context_->data_path();
   FilePath indexed_db_path;
   if (!base_path.empty()) {
     indexed_db_path = base_path.Append(
         IndexedDBContext::kIndexedDBDirectory);
   }
 
   WebSecurityOrigin origin(
       WebSecurityOrigin::createFromDatabaseIdentifier(params.origin));
   GURL url(origin.toString());
 
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   Context()->GetIDBFactory()->deleteDatabase(
       params.name,
       new IndexedDBCallbacks<WebIDBDatabase>(this, params.response_id, url),
       WebSecurityOrigin::createFromDatabaseIdentifier(params.origin), NULL,
       webkit_glue::FilePathToWebString(indexed_db_path));
 }
 
+void IndexedDBDispatcherHost::TransactionComplete(int32 transaction_id) {
+  Context()->TransactionComplete(
+      transaction_dispatcher_host_->transaction_url_map_[transaction_id]);

[michaeln, 2011/08/03 03:40:40]

where are transaction_url_map_ entries erased?

+  transaction_dispatcher_host_->transaction_size_map_.erase(transaction_id);

[michaeln, 2011/08/03 03:40:40]

maybe let TransactionDispatcherHost::OnDestroyed take care of erasing from both
the transaction_url and transaction_size map in all cases

+}
+
 //////////////////////////////////////////////////////////////////////
 // Helper templates.
 //
 
 template <typename ObjectType>
 ObjectType* IndexedDBDispatcherHost::GetOrTerminateProcess(
     IDMap<ObjectType, IDMapOwnPointer>* map, int32 return_object_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   ObjectType* return_object = map->Lookup(return_object_id);
   if (!return_object) {
@@ skipping 96 matching lines @@
   WebIDBTransaction* idb_transaction = parent_->GetOrTerminateProcess(
       &parent_->transaction_dispatcher_host_->map_, params.transaction_id);
   if (!idb_database || !idb_transaction)
     return;
 
   *ec = 0;
   WebIDBObjectStore* object_store = idb_database->createObjectStore(
       params.name, params.key_path, params.auto_increment,
       *idb_transaction, *ec);
   *object_store_id = *ec ? 0 : parent_->Add(object_store);
+  if (parent_->Context()->IsOverQuota(

[michaeln, 2011/08/03 03:40:40]

Is there a WebExceptionCode that means 'quota error'? I'm wondering if this
should fail with a non-zero *ec prior to creating the object store.

[kinuko, 2011/08/03 09:12:22]

QUOTA_EXCEEDED_ERR (22)?

[dgrogan, 2011/08/03 21:45:47]

Same comment as below about what's in the spec.  In short: I'm not sure.

+      database_url_map_[params.idb_database_id])) {
+    idb_transaction->abort();
+  }
 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnDeleteObjectStore(
     int32 idb_database_id,
     const string16& name,
     int32 transaction_id,
     WebKit::WebExceptionCode* ec) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   WebIDBDatabase* idb_database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
@@ skipping 13 matching lines @@
     WebKit::WebExceptionCode* ec) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   WebIDBDatabase* idb_database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
   if (!idb_database)
     return;
 
   *ec = 0;
   idb_database->setVersion(
       version,
-      new IndexedDBCallbacks<WebIDBTransaction>(parent_, response_id),
+      new IndexedDBCallbacks<WebIDBTransaction>(parent_, response_id,
+          database_url_map_[idb_database_id]),
       *ec);
 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnTransaction(
     int32 idb_database_id,
     const std::vector<string16>& names,
     int32 mode,
     int32 timeout,
     int32* idb_transaction_id,
     WebKit::WebExceptionCode* ec) {
   WebIDBDatabase* database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
   if (!database)
       return;
 
   WebDOMStringList object_stores;
   for (std::vector<string16>::const_iterator it = names.begin();
        it != names.end(); ++it) {
     object_stores.append(*it);
   }
 
   *ec = 0;
   WebIDBTransaction* transaction = database->transaction(
       object_stores, mode, timeout, *ec);
   DCHECK(!transaction != !*ec);

[michaeln, 2011/08/03 03:40:40]

unrelated: curious DCHECK :)

-  *idb_transaction_id = *ec ? 0 : parent_->Add(transaction);
+  *idb_transaction_id =
+      *ec ? 0 : parent_->Add(transaction, database_url_map_[idb_database_id]);
 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnOpen(
     int32 idb_database_id, int32 response_id) {
   WebIDBDatabase* database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
   database->open(new IndexedDBDatabaseCallbacks(parent_, response_id));
 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnClose(
     int32 idb_database_id) {
   WebIDBDatabase* database = parent_->GetOrTerminateProcess(
       &map_, idb_database_id);
   database->close();
-  parent_->Context()->quota_manager_proxy()->NotifyStorageAccessed(
-      quota::QuotaClient::kIndexedDatabase, url_map_[idb_database_id],
-      quota::kStorageTypeTemporary);
-  url_map_.erase(idb_database_id);
+  const GURL origin_url = database_url_map_[idb_database_id];
+  database_url_map_.erase(idb_database_id);
+  parent_->Context()->ConnectionClosed(origin_url);

[michaeln, 2011/08/03 03:40:40]

Is OnClose called in all cases or can this method be skipped and OnDestroyed get
called if the object is gc'd?

Would it make sense to move this cleanup code to OnDestroyed? Since that is
where parent_->DestroyObject(&map_, id) is called to remove the WebIDBDatabase
instance from the primary map and delete the instance, i'm wondering if that's a
more appropiate place for this cleanup too.

[dgrogan, 2011/08/04 23:44:55]

Yes it would.  Moved.

[michaeln, 2011/08/05 01:21:47]

Any reason not to move the ConnectionClosed() call to OnDestroyed as well? If it
bother you that the context method is called 'closed', maybe rename it to
ConnectionDestroyed(). The important part is to eventually report deltas in
usage, doing so on close/destroyed time just helps pick up compaction action
that happens asyncly to transaction action. Presuming OnDestroyed is more robust
for map cleanup, why not for this cleanup purpose too? The cleanup logic should
stay together.

[dgrogan, 2011/08/05 03:06:58]

Moved.  Though I'm not sure that OnDestroyed is more robust.  I have no
observations to support that or anything, I really just don't know.

 }
 
 void IndexedDBDispatcherHost::DatabaseDispatcherHost::OnDestroyed(
     int32 object_id) {
   parent_->DestroyObject(&map_, object_id);
 }
 
 
 //////////////////////////////////////////////////////////////////////
 // IndexedDBDispatcherHost::IndexDispatcherHost
@@ skipping 235 matching lines @@
   WebIDBTransaction* idb_transaction = parent_->GetOrTerminateProcess(
       &parent_->transaction_dispatcher_host_->map_, params.transaction_id);
   if (!idb_transaction || !idb_object_store)
     return;
 
   *ec = 0;
   scoped_ptr<WebIDBCallbacks> callbacks(
       new IndexedDBCallbacks<WebIDBKey>(parent_, params.response_id));
   idb_object_store->put(params.serialized_value, params.key, params.put_mode,
                         callbacks.release(), *idb_transaction, *ec);
+  if (*ec)
+    return;
+  int64 size = UTF16ToUTF8(params.serialized_value.data()).size();
+  WebIDBTransactionIDToSizeMap* map =
+      &parent_->transaction_dispatcher_host_->transaction_size_map_;
+  (*map)[params.transaction_id] += size;
 }
 
 void IndexedDBDispatcherHost::ObjectStoreDispatcherHost::OnDelete(
     int idb_object_store_id,
     int32 response_id,
     const IndexedDBKey& key,
     int32 transaction_id,
     WebKit::WebExceptionCode* ec) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   WebIDBObjectStore* idb_object_store = parent_->GetOrTerminateProcess(
@@ skipping 37 matching lines @@
       &map_, params.idb_object_store_id);
   WebIDBTransaction* idb_transaction = parent_->GetOrTerminateProcess(
       &parent_->transaction_dispatcher_host_->map_, params.transaction_id);
   if (!idb_object_store || !idb_transaction)
     return;
 
   *ec = 0;
   WebIDBIndex* index = idb_object_store->createIndex(
       params.name, params.key_path, params.unique, *idb_transaction, *ec);
   *index_id = *ec ? 0 : parent_->Add(index);
+  WebIDBObjectIDToURLMap* transaction_url_map =

[dgrogan, 2011/08/03 01:41:29]

Getting a handle to this variable is just for readability and to make 80-char
linebreaks easier.

+      &parent_->transaction_dispatcher_host_->transaction_url_map_;
+  if (parent_->Context()->IsOverQuota(

[michaeln, 2011/08/03 03:40:40]

Is there a WebExceptionCode that means 'quota error'? I'm wondering if this
should fail with a non-zero *ec prior to creating the index.

[dgrogan, 2011/08/03 21:45:47]

I did it this way because of the following text from the spec.  It's not
precisely on point because we're not asynchronously running into problems. 
Though if we were asynchronously running into problems, we would already have
created and returned an IDBIndex object so the whole paragraph would be moot. 
So I think the way it was intended to be written covers this case.

In some implementations it's possible for the implementation to asynchronously
run into problems creating the index after the createIndex function has
returned. For example in implementations where metadata about the newly created
index is queued up to be inserted into the database asynchronously, or where the
implementation might need to ask the user for permission for quota reasons. Such
implementations must still create and return a IDBIndex object. Instead, once
the implementation realizes that creating the index has failed, it must abort
the transaction using the steps for aborting a transaction. 

+      (*transaction_url_map)[params.transaction_id])) {
+    idb_transaction->abort();
+  }
 }
 
 void IndexedDBDispatcherHost::ObjectStoreDispatcherHost::OnIndex(
     int32 idb_object_store_id,
     const string16& name,
     int32* idb_index_id,
     WebKit::WebExceptionCode* ec) {
   WebIDBObjectStore* idb_object_store = parent_->GetOrTerminateProcess(
       &map_, idb_object_store_id);
   if (!idb_object_store)
@@ skipping 245 matching lines @@
       &map_, transaction_id);
   if (!idb_transaction)
     return;
 
   *ec = 0;
   WebIDBObjectStore* object_store = idb_transaction->objectStore(name, *ec);
   *object_store_id = object_store ? parent_->Add(object_store) : 0;
 }
 
 void IndexedDBDispatcherHost::
     TransactionDispatcherHost::OnDidCompleteTaskEvents(int transaction_id) {

[michaeln, 2011/08/03 03:40:40]

I'm not familiar with these methods or the sequence of events. When does this
get called in the act of a transaction? One-time at the end or multiple times as
progress is made?

[dgrogan, 2011/08/04 23:44:55]

This is called after the success event of each operation in a transaction is
dispatched to JS.

[michaeln, 2011/08/05 01:21:47]

Thnx for the explanation. Sounds like it makes for a great place to monitor size
accumulation!

   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
   WebIDBTransaction* idb_transaction = parent_->GetOrTerminateProcess(
       &map_, transaction_id);
   if (!idb_transaction)
     return;
 
+  if (parent_->Context()->WouldBeOverQuota(
+      transaction_url_map_[transaction_id],
+      transaction_size_map_[transaction_id])) {
+    idb_transaction->abort();

[michaeln, 2011/08/03 03:40:40]

When trans->abort() is called, what does application script see as a result? Is
there any exception/error code associated with the event or callback that is
seen at that script level?

[dgrogan, 2011/08/03 21:45:47]

I don't think so.  This was bothering me as well, that we don't tell the web
author why their transaction suddenly aborted.  The previous quota layout test
didn't indicate that there was any way to know.  I think, as specced, this is
the best we can do.  But _something_ needs to change so that authors know that
the failure is quota related.

[michaeln, 2011/08/04 00:23:27]

Probably worthy of a TODO? We definitely want to indicate to the caller
QUOTA_ERROR in some way.

[dgrogan, 2011/08/04 19:47:50]

Done.

+    return;
+  }
   idb_transaction->didCompleteTaskEvents();
 }
 
 void IndexedDBDispatcherHost::TransactionDispatcherHost::OnDestroyed(
     int32 object_id) {
   parent_->DestroyObject(&map_, object_id);

[michaeln, 2011/08/03 03:40:40]

would this be a good place to erase from the transaction_size_map_ so entries
are cleared on the onAbort() and onTimeout() cases too?

 }