Add blob-writing functionality [as yet un-called] to IDB's backend.

content/browser/indexed_db/indexed_db_backing_store.cc

Index: content/browser/indexed_db/indexed_db_backing_store.cc
diff --git a/content/browser/indexed_db/indexed_db_backing_store.cc b/content/browser/indexed_db/indexed_db_backing_store.cc
index d79407aec9d809ca91a8faf2b8da9ee8c1ee63e5..523dfe6c92755889537276de79b7384050138ad4 100644
--- a/content/browser/indexed_db/indexed_db_backing_store.cc
+++ b/content/browser/indexed_db/indexed_db_backing_store.cc
@@ -11,6 +11,7 @@
 #include "base/logging.h"
 #include "base/metrics/histogram.h"
 #include "base/strings/string_util.h"
+#include "base/strings/stringprintf.h"
 #include "base/strings/utf_string_conversions.h"
 #include "content/browser/child_process_security_policy_impl.h"
 #include "content/browser/indexed_db/indexed_db_blob_info.h"
@@ -26,18 +27,46 @@
 #include "content/common/indexed_db/indexed_db_key.h"
 #include "content/common/indexed_db/indexed_db_key_path.h"
 #include "content/common/indexed_db/indexed_db_key_range.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/url_request/url_request_context.h"
 #include "third_party/WebKit/public/platform/WebIDBTypes.h"
 #include "third_party/WebKit/public/web/WebSerializedScriptValueVersion.h"
 #include "third_party/leveldatabase/env_chromium.h"
 #include "webkit/browser/blob/blob_data_handle.h"
+#include "webkit/browser/fileapi/file_stream_writer.h"
+#include "webkit/browser/fileapi/file_writer_delegate.h"
+#include "webkit/browser/fileapi/local_file_stream_writer.h"
 #include "webkit/common/database/database_identifier.h"
 
+using base::FilePath;
 using base::StringPiece;
+using fileapi::FileWriterDelegate;
 
 namespace content {
 
 namespace {
 
+FilePath GetBlobDirectoryName(const FilePath& pathBase, int64 database_id) {
+  return pathBase.AppendASCII(base::StringPrintf("%lx", database_id));
+}
+
+FilePath GetBlobDirectoryNameForKey(const FilePath& pathBase,
+                                    int64 database_id,
+                                    int64 key) {
+  FilePath path = GetBlobDirectoryName(pathBase, database_id);
+  path = path.AppendASCII(base::StringPrintf(
+      "%02x", static_cast<int>(key & 0x000000000000ff00) >> 8));
+  return path;
+}
+
+// This assumes a file path of dbId/second-to-LSB-of-counter/counter.
+bool MakeIDBBlobDirectory(const FilePath& pathBase,
+                          int64 database_id,
+                          int64 key) {
+  FilePath path = GetBlobDirectoryNameForKey(pathBase, database_id, key);
+  return base::CreateDirectory(path);
+}
+
 static std::string ComputeOriginIdentifier(const GURL& origin_url) {
   return webkit_database::GetIdentifierFromOrigin(origin_url) + "@1";
 }
@@ -441,6 +470,7 @@ IndexedDBBackingStore::IndexedDBBackingStore(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& blob_path,
+    net::URLRequestContext* request_context,
     scoped_ptr<LevelDBDatabase> db,
     scoped_ptr<LevelDBComparator> comparator,
     base::TaskRunner* task_runner)
@@ -448,6 +478,7 @@ IndexedDBBackingStore::IndexedDBBackingStore(
       origin_url_(origin_url),
       blob_path_(blob_path),
       origin_identifier_(ComputeOriginIdentifier(origin_url)),
+      request_context_(request_context),
       task_runner_(task_runner),
       db_(db.Pass()),
       comparator_(comparator.Pass()),
@@ -506,6 +537,7 @@ scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Open(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& path_base,
+    net::URLRequestContext* request_context,
     blink::WebIDBDataLoss* data_loss,
     std::string* data_loss_message,
     bool* disk_full,
@@ -515,6 +547,7 @@ scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Open(
   return IndexedDBBackingStore::Open(indexed_db_factory,
                                      origin_url,
                                      path_base,
+                                     request_context,
                                      data_loss,
                                      data_loss_message,
                                      disk_full,
@@ -655,6 +688,7 @@ scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Open(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& path_base,
+    net::URLRequestContext* request_context,
     blink::WebIDBDataLoss* data_loss,
     std::string* data_loss_message,
     bool* is_disk_full,
@@ -780,6 +814,7 @@ scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Open(
   return Create(indexed_db_factory,
                 origin_url,
                 blob_path,
+                request_context,
                 db.Pass(),
                 comparator.Pass(),
                 task_runner);
@@ -815,6 +850,7 @@ scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::OpenInMemory(
   return Create(NULL /* indexed_db_factory */,
                 origin_url,
                 base::FilePath(),
+                NULL /* request_context */,
                 db.Pass(),
                 comparator.Pass(),
                 task_runner);
@@ -825,15 +861,16 @@ scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Create(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& blob_path,
+    net::URLRequestContext* request_context,
     scoped_ptr<LevelDBDatabase> db,
     scoped_ptr<LevelDBComparator> comparator,
     base::TaskRunner* task_runner) {
   // TODO(jsbell): Handle comparator name changes.
-
   scoped_refptr<IndexedDBBackingStore> backing_store(
       new IndexedDBBackingStore(indexed_db_factory,
                                 origin_url,
                                 blob_path,
+                                request_context,
                                 db.Pass(),
                                 comparator.Pass(),
                                 task_runner));
@@ -1693,6 +1730,198 @@ leveldb::Status IndexedDBBackingStore::KeyExistsInObjectStore(
   return s;
 }
 
+class IndexedDBBackingStore::Transaction::ChainedBlobWriterImpl
+    : public IndexedDBBackingStore::Transaction::ChainedBlobWriter {
+ public:
+  typedef IndexedDBBackingStore::Transaction::WriteDescriptorVec
+      WriteDescriptorVec;
+  ChainedBlobWriterImpl(
+      int64 database_id,
+      IndexedDBBackingStore* backingStore,
+      WriteDescriptorVec& blobs,
+      scoped_refptr<IndexedDBBackingStore::BlobWriteCallback> callback)
+      : waiting_for_callback_(false),
+        database_id_(database_id),
+        backing_store_(backingStore),
+        callback_(callback),
+        aborted_(false) {
+    blobs_.swap(blobs);
+    iter_ = blobs_.begin();
+    WriteNextFile();

[cmumford, 2014/04/18 17:19:06]

By writing the first file in the constructor do you have a problem where the
delegate hasn't yet been set before use?

[ericu, 2014/04/21 20:54:19]

No, the delegate gets set in writeBlobToFileOnIOThread as a result of that
WriteNextFile call.  We create it just as we start to use it.

+  }
+
+  void set_delegate(scoped_ptr<FileWriterDelegate> delegate) {
+    delegate_.reset(delegate.release());
+  }
+
+  void ReportWriteCompletion(bool succeeded, int64 bytes_written) {
+    // TODO(ericu): Check bytes_written against the blob's snapshot value.
+    DCHECK(waiting_for_callback_);
+    DCHECK(!succeeded || bytes_written >= 0);
+    waiting_for_callback_ = false;
+    content::BrowserThread::DeleteSoon(
+        content::BrowserThread::IO, FROM_HERE, delegate_.release());
+    if (aborted_) {
+      self_ref_ = NULL;
+      return;
+    }
+    if (succeeded)
+      WriteNextFile();
+    else
+      callback_->DidFail();
+  }
+
+  void Abort() {
+    if (!waiting_for_callback_)
+      return;
+    self_ref_ = this;
+    aborted_ = true;
+  }
+
+ private:
+  void WriteNextFile() {
+    DCHECK(!waiting_for_callback_);
+    DCHECK(!aborted_);
+    if (iter_ == blobs_.end()) {
+      DCHECK(!self_ref_);
+      callback_->DidSucceed();
+      return;
+    } else {
+      if (!backing_store_->WriteBlobFile(database_id_, *iter_, this)) {
+        callback_->DidFail();
+        return;
+      }
+      waiting_for_callback_ = true;
+      ++iter_;
+    }
+  }
+
+  bool waiting_for_callback_;
+  scoped_refptr<ChainedBlobWriterImpl> self_ref_;

[cmumford, 2014/04/18 17:19:06]

having a self reference seems like a scary way to keep an object alive. Will it
really not outlive the backing store? What about keeping the references in the
backing store itself?

[ericu, 2014/04/21 20:54:19]

It's safe for us to outlive the backing store.  The self_ref_ is only set when
someone tries to abort a running write.  As soon as the write completes, the ref
is cleared without us taking any other action [as the abort has been reported as
complete elsewhere already].  The only purpose of self_ref_ is to let us swallow
that last write completion, even though the transaction that owns us has gone
away.

What I really want is a weak pointer, but unfortunately they're not thread safe.
 That's not a problem in callBlobCallbackOnIDBTaskRunner [it's on the right
task_runner], but would be for the postTask in WriteBlobFile.  I could put
another wrapper there, but I think that would be making things even worse.

I could have the backing store hold a ref in addition to the transaction,
perhaps only when needed, but that makes this less self-contained.  If we ever
wanted to be able to run multiple write transactions at the same time, we'd have
to undo that change.  Granted, I think we're far from there, but I still like
the encapsulation I have here.

If you feel strongly that we shouldn't do a self-ref here, let's talk about it.

+  WriteDescriptorVec blobs_;
+  WriteDescriptorVec::const_iterator iter_;
+  int64 database_id_;
+  IndexedDBBackingStore* backing_store_;
+  scoped_refptr<IndexedDBBackingStore::BlobWriteCallback> callback_;
+  scoped_ptr<FileWriterDelegate> delegate_;
+  bool aborted_;
+};
+
+class LocalWriteClosure : public FileWriterDelegate::DelegateWriteCallback,
+                          public base::RefCounted<LocalWriteClosure> {
+ public:
+  LocalWriteClosure(IndexedDBBackingStore::Transaction::ChainedBlobWriter*
+                        chained_blob_writer_,
+                    base::TaskRunner* task_runner)
+      : chained_blob_writer_(chained_blob_writer_),
+        task_runner_(task_runner),
+        bytes_written_(-1) {}
+
+  void Run(base::File::Error rv,
+           int64 bytes,
+           FileWriterDelegate::WriteProgressStatus write_status) {
+    if (write_status == FileWriterDelegate::SUCCESS_IO_PENDING)
+      return;  // We don't care about progress events.
+    if (rv == base::File::FILE_OK) {
+      DCHECK(bytes >= 0);
+      DCHECK(write_status == FileWriterDelegate::SUCCESS_COMPLETED);
+      bytes_written_ = bytes;
+    } else {
+      DCHECK(write_status == FileWriterDelegate::ERROR_WRITE_STARTED ||
+             write_status == FileWriterDelegate::ERROR_WRITE_NOT_STARTED);
+    }
+    task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&LocalWriteClosure::callBlobCallbackOnIDBTaskRunner,
+                   this,
+                   write_status == FileWriterDelegate::SUCCESS_COMPLETED));
+  }
+
+  void writeBlobToFileOnIOThread(const FilePath& file_path,
+                                 const GURL& blob_url,
+                                 net::URLRequestContext* request_context) {
+    DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::IO));
+    scoped_ptr<fileapi::FileStreamWriter> writer(
+        fileapi::FileStreamWriter::CreateForLocalFile(
+            task_runner_, file_path, 0,
+            fileapi::FileStreamWriter::CREATE_NEW_FILE));
+    scoped_ptr<FileWriterDelegate> delegate(
+        new FileWriterDelegate(writer.Pass()));
+
+    DCHECK(blob_url.is_valid());
+    scoped_ptr<net::URLRequest> blob_request(request_context->CreateRequest(
+        blob_url, net::DEFAULT_PRIORITY, delegate.get(), NULL));
+
+    delegate->Start(blob_request.Pass(),
+                    base::Bind(&LocalWriteClosure::Run, this));
+    chained_blob_writer_->set_delegate(delegate.Pass());
+  }
+
+ private:
+  void callBlobCallbackOnIDBTaskRunner(bool succeeded) {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+    chained_blob_writer_->ReportWriteCompletion(succeeded, bytes_written_);
+  }
+
+  IndexedDBBackingStore::Transaction::ChainedBlobWriter* chained_blob_writer_;
+  base::TaskRunner* task_runner_;
+  int64 bytes_written_;
+};
+
+bool IndexedDBBackingStore::WriteBlobFile(
+    int64 database_id,
+    const Transaction::WriteDescriptor& descriptor,
+    Transaction::ChainedBlobWriter* chained_blob_writer) {
+
+  if (!MakeIDBBlobDirectory(blob_path_, database_id, descriptor.key()))
+    return false;

[cmumford, 2014/04/18 17:19:06]

If we can't create the dir (or copy the file below) why aren't we calling
ReportWriteCompletion(..., false, ...)?

[ericu, 2014/04/21 20:54:19]

We return false, and WriteNextFile [our only caller] calls the failure callback
for us.  ReportWriteCompletion is for writes that got further [completed File
copies or Blobs where we created a delegate_ and tried to Start it].

I was going to say it was just for cases where we had a delegate_, but then
noticed the File case didn't.  I've added an if to short-circuit the DeleteSoon
in that case.  I think it's correct either way, but cleaner and less computation
this way.

+
+  FilePath path = GetBlobFileName(database_id, descriptor.key());
+
+  if (descriptor.is_file()) {
+    DCHECK(!descriptor.file_path().empty());
+    if (!base::CopyFile(descriptor.file_path(), path))
+      return false;
+
+    base::File::Info info;
+    if (base::GetFileInfo(descriptor.file_path(), &info)) {
+      // TODO(ericu): Validate the snapshot date here.  Expand WriteDescriptor
+      // to include snapshot date and file size, and check both.
+      if (!base::TouchFile(path, info.last_accessed, info.last_modified))
+        ;  // TODO(ericu): Complain quietly; timestamp's probably not vital.
+    } else {
+      ;  // TODO(ericu): Complain quietly; timestamp's probably not vital.
+    }
+
+    task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&Transaction::ChainedBlobWriter::ReportWriteCompletion,
+                   chained_blob_writer,
+                   true,
+                   info.size));
+  } else {
+    DCHECK(descriptor.url().is_valid());
+    scoped_refptr<LocalWriteClosure> write_closure(
+        new LocalWriteClosure(chained_blob_writer, task_runner_));
+    content::BrowserThread::PostTask(
+        content::BrowserThread::IO,
+        FROM_HERE,
+        base::Bind(&LocalWriteClosure::writeBlobToFileOnIOThread,
+                   write_closure.get(),
+                   path,
+                   descriptor.url(),
+                   request_context_));
+  }
+  return true;
+}
+
+// This assumes a file path of dbId/second-to-LSB-of-counter/counter.
+FilePath IndexedDBBackingStore::GetBlobFileName(int64 database_id, int64 key) {
+  FilePath path = GetBlobDirectoryNameForKey(blob_path_, database_id, key);
+  path = path.AppendASCII(base::StringPrintf("%lx", key));
+  return path;
+}
+
 static bool CheckIndexAndMetaDataKey(const LevelDBIterator* it,
                                      const std::string& stop_key,
                                      int64 index_id,
@@ -1809,6 +2038,16 @@ leveldb::Status IndexedDBBackingStore::GetIndexes(
   return s;
 }
 
+bool IndexedDBBackingStore::RemoveBlobFile(int64 database_id, int64 key) {
+  FilePath fileName = GetBlobFileName(database_id, key);
+  return base::DeleteFile(fileName, false);
+}
+
+bool IndexedDBBackingStore::RemoveBlobDirectory(int64 database_id) {
+  FilePath dirName = GetBlobDirectoryName(blob_path_, database_id);
+  return base::DeleteFile(dirName, true);
+}
+
 WARN_UNUSED_RESULT static leveldb::Status SetMaxIndexId(
     LevelDBTransaction* transaction,
     int64 database_id,
@@ -2946,9 +3185,60 @@ leveldb::Status IndexedDBBackingStore::Transaction::Commit() {
   return s;
 }
 
+
+class IndexedDBBackingStore::Transaction::BlobWriteCallbackWrapper
+    : public IndexedDBBackingStore::BlobWriteCallback {
+ public:
+  BlobWriteCallbackWrapper(IndexedDBBackingStore::Transaction* transaction,
+                           scoped_refptr<BlobWriteCallback> callback)
+      : transaction_(transaction), callback_(callback) {}
+  virtual void DidSucceed() {
+    callback_->DidSucceed();
+    transaction_->chained_blob_writer_ = NULL;
+  }
+  virtual void DidFail() {
+    callback_->DidFail();
+    transaction_->chained_blob_writer_ = NULL;
+  }
+
+ private:
+  IndexedDBBackingStore::Transaction* transaction_;
+  scoped_refptr<BlobWriteCallback> callback_;
+};
+
+void IndexedDBBackingStore::Transaction::WriteNewBlobs(
+    BlobEntryKeyValuePairVec& new_blob_entries,
+    WriteDescriptorVec& new_files_to_write,
+    scoped_refptr<BlobWriteCallback> callback) {
+  DCHECK_GT(new_files_to_write.size(), 0UL);
+  DCHECK_GT(database_id_, 0);
+  BlobEntryKeyValuePairVec::iterator blob_entry_iter;
+  for (blob_entry_iter = new_blob_entries.begin();
+       blob_entry_iter != new_blob_entries.end();
+       ++blob_entry_iter) {
+    // Add the new blob-table entry for each blob to the main transaction, or
+    // remove any entry that may exist if there's no new one.
+    if (!blob_entry_iter->second.size())
+      transaction_->Remove(blob_entry_iter->first.Encode());
+    else
+      transaction_->Put(blob_entry_iter->first.Encode(),
+                        &blob_entry_iter->second);
+  }
+  // Creating the writer will start it going asynchronously.
+  chained_blob_writer_ =
+      new ChainedBlobWriterImpl(database_id_,
+                                backing_store_,
+                                new_files_to_write,
+                                new BlobWriteCallbackWrapper(this, callback));
+}
+
 void IndexedDBBackingStore::Transaction::Rollback() {
   IDB_TRACE("IndexedDBBackingStore::Transaction::Rollback");
   DCHECK(transaction_.get());
+  if (chained_blob_writer_) {
+    chained_blob_writer_->Abort();
+    chained_blob_writer_ = NULL;
+  }
   transaction_->Rollback();
   transaction_ = NULL;
 }
@@ -3004,4 +3294,14 @@ void IndexedDBBackingStore::Transaction::PutBlobInfo(
   DCHECK(!handles || !handles->size());
 }
 
+IndexedDBBackingStore::Transaction::WriteDescriptor::WriteDescriptor(
+    const GURL& url,
+    int64_t key)
+    : is_file_(false), url_(url), key_(key) {}
+
+IndexedDBBackingStore::Transaction::WriteDescriptor::WriteDescriptor(
+    const FilePath& file_path,
+    int64_t key)
+    : is_file_(true), file_path_(file_path), key_(key) {}
+
 }  // namespace content