Add blob-writing functionality [as yet un-called] to IDB's backend.

content/browser/indexed_db/indexed_db_backing_store.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/indexed_db/indexed_db_backing_store.h"
 
 #include "base/file_util.h"
 #include "base/files/file_path.h"
 #include "base/json/json_reader.h"
 #include "base/json/json_writer.h"
 #include "base/logging.h"
 #include "base/metrics/histogram.h"
 #include "base/strings/string_util.h"
+#include "base/strings/stringprintf.h"
 #include "base/strings/utf_string_conversions.h"
 #include "content/browser/child_process_security_policy_impl.h"
 #include "content/browser/indexed_db/indexed_db_blob_info.h"
 #include "content/browser/indexed_db/indexed_db_database_error.h"
 #include "content/browser/indexed_db/indexed_db_leveldb_coding.h"
 #include "content/browser/indexed_db/indexed_db_metadata.h"
 #include "content/browser/indexed_db/indexed_db_tracing.h"
 #include "content/browser/indexed_db/indexed_db_value.h"
 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
 #include "content/browser/indexed_db/leveldb/leveldb_transaction.h"
 #include "content/common/indexed_db/indexed_db_key.h"
 #include "content/common/indexed_db/indexed_db_key_path.h"
 #include "content/common/indexed_db/indexed_db_key_range.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/url_request/url_request_context.h"
 #include "third_party/WebKit/public/platform/WebIDBTypes.h"
 #include "third_party/WebKit/public/web/WebSerializedScriptValueVersion.h"
 #include "third_party/leveldatabase/env_chromium.h"
 #include "webkit/browser/blob/blob_data_handle.h"
+#include "webkit/browser/fileapi/file_stream_writer.h"
+#include "webkit/browser/fileapi/file_writer_delegate.h"
+#include "webkit/browser/fileapi/local_file_stream_writer.h"
 #include "webkit/common/database/database_identifier.h"
 
+using base::FilePath;
 using base::StringPiece;
+using fileapi::FileWriterDelegate;
 
 namespace content {
 
 namespace {
 
+FilePath GetBlobDirectoryName(const FilePath& pathBase, int64 database_id) {
+  return pathBase.AppendASCII(base::StringPrintf("%lx", database_id));
+}
+
+FilePath GetBlobDirectoryNameForKey(const FilePath& pathBase,
+                                    int64 database_id,
+                                    int64 key) {
+  FilePath path = GetBlobDirectoryName(pathBase, database_id);
+  path = path.AppendASCII(base::StringPrintf(
+      "%02x", static_cast<int>(key & 0x000000000000ff00) >> 8));
+  return path;
+}
+
+// This assumes a file path of dbId/second-to-LSB-of-counter/counter.
+bool MakeIDBBlobDirectory(const FilePath& pathBase,
+                          int64 database_id,
+                          int64 key) {
+  FilePath path = GetBlobDirectoryNameForKey(pathBase, database_id, key);
+  return base::CreateDirectory(path);
+}
+
 static std::string ComputeOriginIdentifier(const GURL& origin_url) {
   return webkit_database::GetIdentifierFromOrigin(origin_url) + "@1";
 }
 
 static base::FilePath ComputeFileName(const GURL& origin_url) {
   return base::FilePath()
       .AppendASCII(webkit_database::GetIdentifierFromOrigin(origin_url))
       .AddExtension(FILE_PATH_LITERAL(".indexeddb.leveldb"));
 }
 
@@ skipping 383 matching lines @@
   virtual leveldb::Status DestroyLevelDB(const base::FilePath& file_name)
       OVERRIDE {
     return LevelDBDatabase::Destroy(file_name);
   }
 };
 
 IndexedDBBackingStore::IndexedDBBackingStore(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& blob_path,
+    net::URLRequestContext* request_context,
     scoped_ptr<LevelDBDatabase> db,
     scoped_ptr<LevelDBComparator> comparator,
     base::TaskRunner* task_runner)
     : indexed_db_factory_(indexed_db_factory),
       origin_url_(origin_url),
       blob_path_(blob_path),
       origin_identifier_(ComputeOriginIdentifier(origin_url)),
+      request_context_(request_context),
       task_runner_(task_runner),
       db_(db.Pass()),
       comparator_(comparator.Pass()),
       active_blob_registry_(this) {}
 
 IndexedDBBackingStore::~IndexedDBBackingStore() {
   if (!blob_path_.empty() && !child_process_ids_granted_.empty()) {
     ChildProcessSecurityPolicyImpl* policy =
         ChildProcessSecurityPolicyImpl::GetInstance();
     std::set<int>::const_iterator iter;
@@ skipping 38 matching lines @@
   INDEXED_DB_BACKING_STORE_OPEN_NO_RECOVERY,
   INDEXED_DB_BACKING_STORE_OPEN_FAILED_PRIOR_CORRUPTION,
   INDEXED_DB_BACKING_STORE_OPEN_MAX,
 };
 
 // static
 scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Open(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& path_base,
+    net::URLRequestContext* request_context,
     blink::WebIDBDataLoss* data_loss,
     std::string* data_loss_message,
     bool* disk_full,
     base::TaskRunner* task_runner) {
   *data_loss = blink::WebIDBDataLossNone;
   DefaultLevelDBFactory leveldb_factory;
   return IndexedDBBackingStore::Open(indexed_db_factory,
                                      origin_url,
                                      path_base,
+                                     request_context,
                                      data_loss,
                                      data_loss_message,
                                      disk_full,
                                      &leveldb_factory,
                                      task_runner);
 }
 
 static std::string OriginToCustomHistogramSuffix(const GURL& origin_url) {
   if (origin_url.host() == "docs.google.com")
     return ".Docs";
@@ skipping 120 matching lines @@
     return false;
   int written = file.Write(0, output_js.c_str(), output_js.length());
   return size_t(written) == output_js.length();
 }
 
 // static
 scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Open(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& path_base,
+    net::URLRequestContext* request_context,
     blink::WebIDBDataLoss* data_loss,
     std::string* data_loss_message,
     bool* is_disk_full,
     LevelDBFactory* leveldb_factory,
     base::TaskRunner* task_runner) {
   IDB_TRACE("IndexedDBBackingStore::Open");
   DCHECK(!path_base.empty());
   *data_loss = blink::WebIDBDataLossNone;
   *data_loss_message = "";
   *is_disk_full = false;
@@ skipping 105 matching lines @@
   if (!db) {
     NOTREACHED();
     HistogramOpenStatus(INDEXED_DB_BACKING_STORE_OPEN_FAILED_UNKNOWN_ERR,
                         origin_url);
     return scoped_refptr<IndexedDBBackingStore>();
   }
 
   return Create(indexed_db_factory,
                 origin_url,
                 blob_path,
+                request_context,
                 db.Pass(),
                 comparator.Pass(),
                 task_runner);
 }
 
 // static
 scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::OpenInMemory(
     const GURL& origin_url,
     base::TaskRunner* task_runner) {
   DefaultLevelDBFactory leveldb_factory;
@@ skipping 15 matching lines @@
     LOG(ERROR) << "LevelDBDatabase::OpenInMemory failed.";
     HistogramOpenStatus(INDEXED_DB_BACKING_STORE_OPEN_MEMORY_FAILED,
                         origin_url);
     return scoped_refptr<IndexedDBBackingStore>();
   }
   HistogramOpenStatus(INDEXED_DB_BACKING_STORE_OPEN_MEMORY_SUCCESS, origin_url);
 
   return Create(NULL /* indexed_db_factory */,
                 origin_url,
                 base::FilePath(),
+                NULL /* request_context */,
                 db.Pass(),
                 comparator.Pass(),
                 task_runner);
 }
 
 // static
 scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::Create(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& blob_path,
+    net::URLRequestContext* request_context,
     scoped_ptr<LevelDBDatabase> db,
     scoped_ptr<LevelDBComparator> comparator,
     base::TaskRunner* task_runner) {
   // TODO(jsbell): Handle comparator name changes.
-
   scoped_refptr<IndexedDBBackingStore> backing_store(
       new IndexedDBBackingStore(indexed_db_factory,
                                 origin_url,
                                 blob_path,
+                                request_context,
                                 db.Pass(),
                                 comparator.Pass(),
                                 task_runner));
   if (!SetUpMetadata(backing_store->db_.get(),
                      backing_store->origin_identifier_))
     return scoped_refptr<IndexedDBBackingStore>();
 
   return backing_store;
 }
 
@@ skipping 839 matching lines @@
   StringPiece slice(data);
   if (!DecodeVarInt(&slice, &version))
     return InternalInconsistencyStatus();
 
   std::string encoded_key;
   EncodeIDBKey(key, &encoded_key);
   found_record_identifier->Reset(encoded_key, version);
   return s;
 }
 
+class IndexedDBBackingStore::Transaction::ChainedBlobWriterImpl
+    : public IndexedDBBackingStore::Transaction::ChainedBlobWriter {
+ public:
+  typedef IndexedDBBackingStore::Transaction::WriteDescriptorVec
+      WriteDescriptorVec;
+  ChainedBlobWriterImpl(
+      int64 database_id,
+      IndexedDBBackingStore* backingStore,
+      WriteDescriptorVec& blobs,
+      scoped_refptr<IndexedDBBackingStore::BlobWriteCallback> callback)
+      : waiting_for_callback_(false),
+        database_id_(database_id),
+        backing_store_(backingStore),
+        callback_(callback),
+        aborted_(false) {
+    blobs_.swap(blobs);
+    iter_ = blobs_.begin();
+    WriteNextFile();
+  }
+
+  void set_delegate(scoped_ptr<FileWriterDelegate> delegate) {
+    delegate_.reset(delegate.release());
+  }
+
+  void ReportWriteCompletion(bool succeeded, int64 bytes_written) {
+    // TODO(ericu): Check bytes_written against the blob's snapshot value.
+    DCHECK(waiting_for_callback_);
+    DCHECK(!succeeded || bytes_written >= 0);
+    waiting_for_callback_ = false;
+    if (delegate_.get())  // Only present for Blob, not File.
+      content::BrowserThread::DeleteSoon(
+          content::BrowserThread::IO, FROM_HERE, delegate_.release());
+    if (aborted_) {
+      self_ref_ = NULL;
+      return;
+    }
+    if (succeeded)
+      WriteNextFile();
+    else
+      callback_->Run(false);
+  }
+
+  void Abort() {
+    if (!waiting_for_callback_)
+      return;
+    self_ref_ = this;
+    aborted_ = true;
+  }
+
+ private:
+  void WriteNextFile() {
+    DCHECK(!waiting_for_callback_);
+    DCHECK(!aborted_);
+    if (iter_ == blobs_.end()) {
+      DCHECK(!self_ref_);
+      callback_->Run(true);
+      return;
+    } else {
+      if (!backing_store_->WriteBlobFile(database_id_, *iter_, this)) {
+        callback_->Run(false);
+        return;
+      }
+      waiting_for_callback_ = true;
+      ++iter_;
+    }
+  }
+
+  bool waiting_for_callback_;
+  scoped_refptr<ChainedBlobWriterImpl> self_ref_;
+  WriteDescriptorVec blobs_;
+  WriteDescriptorVec::const_iterator iter_;
+  int64 database_id_;
+  IndexedDBBackingStore* backing_store_;
+  scoped_refptr<IndexedDBBackingStore::BlobWriteCallback> callback_;
+  scoped_ptr<FileWriterDelegate> delegate_;
+  bool aborted_;
+};
+
+class LocalWriteClosure : public FileWriterDelegate::DelegateWriteCallback,
+                          public base::RefCounted<LocalWriteClosure> {
+ public:
+  LocalWriteClosure(IndexedDBBackingStore::Transaction::ChainedBlobWriter*
+                        chained_blob_writer_,
+                    base::TaskRunner* task_runner)
+      : chained_blob_writer_(chained_blob_writer_),
+        task_runner_(task_runner),
+        bytes_written_(-1) {}
+
+  void Run(base::File::Error rv,
+           int64 bytes,
+           FileWriterDelegate::WriteProgressStatus write_status) {
+    if (write_status == FileWriterDelegate::SUCCESS_IO_PENDING)
+      return;  // We don't care about progress events.
+    if (rv == base::File::FILE_OK) {
+      DCHECK(bytes >= 0);
+      DCHECK(write_status == FileWriterDelegate::SUCCESS_COMPLETED);
+      bytes_written_ = bytes;
+    } else {
+      DCHECK(write_status == FileWriterDelegate::ERROR_WRITE_STARTED ||
+             write_status == FileWriterDelegate::ERROR_WRITE_NOT_STARTED);
+    }
+    task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&LocalWriteClosure::callBlobCallbackOnIDBTaskRunner,
+                   this,
+                   write_status == FileWriterDelegate::SUCCESS_COMPLETED));
+  }
+
+  void writeBlobToFileOnIOThread(const FilePath& file_path,
+                                 const GURL& blob_url,
+                                 net::URLRequestContext* request_context) {
+    DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::IO));
+    scoped_ptr<fileapi::FileStreamWriter> writer(
+        fileapi::FileStreamWriter::CreateForLocalFile(
+            task_runner_, file_path, 0,
+            fileapi::FileStreamWriter::CREATE_NEW_FILE));
+    scoped_ptr<FileWriterDelegate> delegate(
+        new FileWriterDelegate(writer.Pass()));
+
+    DCHECK(blob_url.is_valid());
+    scoped_ptr<net::URLRequest> blob_request(request_context->CreateRequest(
+        blob_url, net::DEFAULT_PRIORITY, delegate.get(), NULL));
+
+    delegate->Start(blob_request.Pass(),
+                    base::Bind(&LocalWriteClosure::Run, this));
+    chained_blob_writer_->set_delegate(delegate.Pass());
+  }
+
+ private:
+  void callBlobCallbackOnIDBTaskRunner(bool succeeded) {
+    DCHECK(task_runner_->RunsTasksOnCurrentThread());
+    chained_blob_writer_->ReportWriteCompletion(succeeded, bytes_written_);
+  }
+
+  IndexedDBBackingStore::Transaction::ChainedBlobWriter* chained_blob_writer_;
+  base::TaskRunner* task_runner_;
+  int64 bytes_written_;
+};
+
+bool IndexedDBBackingStore::WriteBlobFile(
+    int64 database_id,
+    const Transaction::WriteDescriptor& descriptor,
+    Transaction::ChainedBlobWriter* chained_blob_writer) {
+
+  if (!MakeIDBBlobDirectory(blob_path_, database_id, descriptor.key()))
+    return false;
+
+  FilePath path = GetBlobFileName(database_id, descriptor.key());
+
+  if (descriptor.is_file()) {
+    DCHECK(!descriptor.file_path().empty());
+    if (!base::CopyFile(descriptor.file_path(), path))
+      return false;
+
+    base::File::Info info;
+    if (base::GetFileInfo(descriptor.file_path(), &info)) {
+      // TODO(ericu): Validate the snapshot date here.  Expand WriteDescriptor
+      // to include snapshot date and file size, and check both.
+      if (!base::TouchFile(path, info.last_accessed, info.last_modified))
+        ;  // TODO(ericu): Complain quietly; timestamp's probably not vital.
+    } else {
+      ;  // TODO(ericu): Complain quietly; timestamp's probably not vital.
+    }
+
+    task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&Transaction::ChainedBlobWriter::ReportWriteCompletion,
+                   chained_blob_writer,
+                   true,
+                   info.size));
+  } else {
+    DCHECK(descriptor.url().is_valid());
+    scoped_refptr<LocalWriteClosure> write_closure(
+        new LocalWriteClosure(chained_blob_writer, task_runner_));
+    content::BrowserThread::PostTask(
+        content::BrowserThread::IO,
+        FROM_HERE,
+        base::Bind(&LocalWriteClosure::writeBlobToFileOnIOThread,
+                   write_closure.get(),
+                   path,
+                   descriptor.url(),
+                   request_context_));
+  }
+  return true;
+}
+
+// This assumes a file path of dbId/second-to-LSB-of-counter/counter.

[jsbell, 2014/04/24 00:16:13]

Just to keep assumptions together at the top of the file, maybe factor out a
helper GetBlobFilenameForKey(blob_path_, database_id, key) ?

[ericu, 2014/04/28 17:49:04]

Done.

+FilePath IndexedDBBackingStore::GetBlobFileName(int64 database_id, int64 key) {
+  FilePath path = GetBlobDirectoryNameForKey(blob_path_, database_id, key);
+  path = path.AppendASCII(base::StringPrintf("%lx", key));
+  return path;
+}
+
 static bool CheckIndexAndMetaDataKey(const LevelDBIterator* it,
                                      const std::string& stop_key,
                                      int64 index_id,
                                      unsigned char meta_data_type) {
   if (!it->IsValid() || CompareKeys(it->Key(), stop_key) >= 0)
     return false;
 
   StringPiece slice(it->Key());
   IndexMetaDataKey meta_data_key;
   bool ok = IndexMetaDataKey::Decode(&slice, &meta_data_key);
@@ skipping 96 matching lines @@
     (*indexes)[index_id] = IndexedDBIndexMetadata(
         index_name, index_id, key_path, index_unique, index_multi_entry);
   }
 
   if (!s.ok())
     INTERNAL_READ_ERROR_UNTESTED(GET_INDEXES);
 
   return s;
 }
 
+bool IndexedDBBackingStore::RemoveBlobFile(int64 database_id, int64 key) {
+  FilePath fileName = GetBlobFileName(database_id, key);
+  return base::DeleteFile(fileName, false);
+}
+
+bool IndexedDBBackingStore::RemoveBlobDirectory(int64 database_id) {
+  FilePath dirName = GetBlobDirectoryName(blob_path_, database_id);
+  return base::DeleteFile(dirName, true);
+}
+
 WARN_UNUSED_RESULT static leveldb::Status SetMaxIndexId(
     LevelDBTransaction* transaction,
     int64 database_id,
     int64 object_store_id,
     int64 index_id) {
   int64 max_index_id = -1;
   const std::string max_index_id_key = ObjectStoreMetaDataKey::Encode(
       database_id, object_store_id, ObjectStoreMetaDataKey::MAX_INDEX_ID);
   bool found = false;
   leveldb::Status s =
@@ skipping 1117 matching lines @@
 leveldb::Status IndexedDBBackingStore::Transaction::Commit() {
   IDB_TRACE("IndexedDBBackingStore::Transaction::Commit");
   DCHECK(transaction_.get());
   leveldb::Status s = transaction_->Commit();
   transaction_ = NULL;
   if (!s.ok())
     INTERNAL_WRITE_ERROR_UNTESTED(TRANSACTION_COMMIT_METHOD);
   return s;
 }
 
+
+class IndexedDBBackingStore::Transaction::BlobWriteCallbackWrapper
+    : public IndexedDBBackingStore::BlobWriteCallback {
+ public:
+  BlobWriteCallbackWrapper(IndexedDBBackingStore::Transaction* transaction,
+                           scoped_refptr<BlobWriteCallback> callback)
+      : transaction_(transaction), callback_(callback) {}
+  virtual void Run(bool succeeded) {
+    callback_->Run(succeeded);
+    transaction_->chained_blob_writer_ = NULL;
+  }
+
+ private:
+  IndexedDBBackingStore::Transaction* transaction_;
+  scoped_refptr<BlobWriteCallback> callback_;
+};
+
+void IndexedDBBackingStore::Transaction::WriteNewBlobs(
+    BlobEntryKeyValuePairVec& new_blob_entries,
+    WriteDescriptorVec& new_files_to_write,
+    scoped_refptr<BlobWriteCallback> callback) {
+  DCHECK_GT(new_files_to_write.size(), 0UL);
+  DCHECK_GT(database_id_, 0);
+  BlobEntryKeyValuePairVec::iterator blob_entry_iter;
+  for (blob_entry_iter = new_blob_entries.begin();
+       blob_entry_iter != new_blob_entries.end();
+       ++blob_entry_iter) {
+    // Add the new blob-table entry for each blob to the main transaction, or
+    // remove any entry that may exist if there's no new one.
+    if (!blob_entry_iter->second.size())
+      transaction_->Remove(blob_entry_iter->first.Encode());
+    else
+      transaction_->Put(blob_entry_iter->first.Encode(),
+                        &blob_entry_iter->second);
+  }
+  // Creating the writer will start it going asynchronously.
+  chained_blob_writer_ =
+      new ChainedBlobWriterImpl(database_id_,
+                                backing_store_,
+                                new_files_to_write,
+                                new BlobWriteCallbackWrapper(this, callback));
+}
+
 void IndexedDBBackingStore::Transaction::Rollback() {
   IDB_TRACE("IndexedDBBackingStore::Transaction::Rollback");
   DCHECK(transaction_.get());
+  if (chained_blob_writer_) {
+    chained_blob_writer_->Abort();
+    chained_blob_writer_ = NULL;
+  }
   transaction_->Rollback();
   transaction_ = NULL;
 }
 
 IndexedDBBackingStore::Transaction::BlobChangeRecord::BlobChangeRecord(
     const std::string& key, int64 object_store_id)
     : key_(key), object_store_id_(object_store_id) {
 }
 
 IndexedDBBackingStore::Transaction::BlobChangeRecord::~BlobChangeRecord() {
@@ skipping 35 matching lines @@
     blob_change_map_[key] = record;
   } else {
     record = it->second;
   }
   DCHECK_EQ(record->object_store_id(), object_store_id);
   record->SetBlobInfo(blob_info);
   record->SetHandles(handles);
   DCHECK(!handles || !handles->size());
 }
 
+IndexedDBBackingStore::Transaction::WriteDescriptor::WriteDescriptor(
+    const GURL& url,
+    int64_t key)
+    : is_file_(false), url_(url), key_(key) {}
+
+IndexedDBBackingStore::Transaction::WriteDescriptor::WriteDescriptor(
+    const FilePath& file_path,
+    int64_t key)
+    : is_file_(true), file_path_(file_path), key_(key) {}
+
 }  // namespace content