Add most of the metadata-handling code for blobs. It's not quite all there, but

content/browser/indexed_db/indexed_db_backing_store.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/indexed_db/indexed_db_backing_store.h"
 
 #include "base/file_util.h"
 #include "base/files/file_path.h"
 #include "base/format_macros.h"
 #include "base/json/json_reader.h"
@@ skipping 110 matching lines @@
   GET_NEW_DATABASE_ID,
   GET_NEW_VERSION_NUMBER,
   CREATE_IDBDATABASE_METADATA,
   DELETE_DATABASE,
   TRANSACTION_COMMIT_METHOD,  // TRANSACTION_COMMIT is a WinNT.h macro
   GET_DATABASE_NAMES,
   DELETE_INDEX,
   CLEAR_OBJECT_STORE,
   READ_BLOB_JOURNAL,
   DECODE_BLOB_JOURNAL,
+  GET_BLOB_KEY_GENERATOR_CURRENT_NUMBER,
+  GET_BLOB_INFO_FOR_RECORD,
   INTERNAL_ERROR_MAX,
 };
 
 static void RecordInternalError(const char* type,
                                 IndexedDBBackingStoreErrorSource location) {
   std::string name;
   name.append("WebCore.IndexedDB.BackingStore.").append(type).append("Error");
   base::Histogram::FactoryGet(name,
                               1,
                               INTERNAL_ERROR_MAX,
@@ skipping 156 matching lines @@
   return content::Compare(a, b, false /*index_keys*/);
 }
 
 const char* IndexedDBBackingStore::Comparator::Name() const {
   return "idb_cmp1";
 }
 
 // 0 - Initial version.
 // 1 - Adds UserIntVersion to DatabaseMetaData.
 // 2 - Adds DataVersion to to global metadata.
-static const int64 kLatestKnownSchemaVersion = 2;
+// 3 - Adds metadata needed for blob support.
+static const int64 kLatestKnownSchemaVersion = 3;
 WARN_UNUSED_RESULT static bool IsSchemaKnown(LevelDBDatabase* db, bool* known) {
   int64 db_schema_version = 0;
   bool found = false;
   leveldb::Status s =
       GetInt(db, SchemaVersionKey::Encode(), &db_schema_version, &found);
   if (!s.ok())
     return false;
   if (!found) {
     *known = true;
     return true;
@@ skipping 16 matching lines @@
 
   if (db_data_version > latest_known_data_version) {
     *known = false;
     return true;
   }
 
   *known = true;
   return true;
 }
 
-WARN_UNUSED_RESULT static bool SetUpMetadata(
-    LevelDBDatabase* db,
-    const std::string& origin_identifier) {
+// TODO(ericu): Move this down into the member section of this file.  I'm
+// leaving it here for this CL as it's easier to see the diffs in place.
+WARN_UNUSED_RESULT bool IndexedDBBackingStore::SetUpMetadata() {
   const uint32 latest_known_data_version =
       blink::kSerializedScriptValueVersion;
   const std::string schema_version_key = SchemaVersionKey::Encode();
   const std::string data_version_key = DataVersionKey::Encode();
 
-  scoped_refptr<LevelDBTransaction> transaction = new LevelDBTransaction(db);
+  scoped_refptr<LevelDBTransaction> transaction =
+      new LevelDBTransaction(db_.get());
 
   int64 db_schema_version = 0;
   int64 db_data_version = 0;
   bool found = false;
   leveldb::Status s =
       GetInt(transaction.get(), schema_version_key, &db_schema_version, &found);
   if (!s.ok()) {
     INTERNAL_READ_ERROR_UNTESTED(SET_UP_METADATA);
     return false;
   }
   if (!found) {
     // Initialize new backing store.
     db_schema_version = kLatestKnownSchemaVersion;
     PutInt(transaction.get(), schema_version_key, db_schema_version);
     db_data_version = latest_known_data_version;
     PutInt(transaction.get(), data_version_key, db_data_version);
+    // If a blob directory already exists for this database, blow it away.  It's
+    // leftover from a partially-purged previous generation of data.
+    if (!base::DeleteFile(blob_path_, true)) {
+      INTERNAL_WRITE_ERROR(SET_UP_METADATA);

[cmumford, 2014/05/06 18:33:11]

INTERNAL_READ_ERROR_UNTESTED

[ericu, 2014/05/06 20:07:42]

INTERNAL_WRITE_ERROR_UNTESTED?  Switched to that.

+      return false;
+    }
   } else {
     // Upgrade old backing store.
     DCHECK_LE(db_schema_version, kLatestKnownSchemaVersion);
     if (db_schema_version < 1) {
       db_schema_version = 1;
       PutInt(transaction.get(), schema_version_key, db_schema_version);
       const std::string start_key =
-          DatabaseNameKey::EncodeMinKeyForOrigin(origin_identifier);
+          DatabaseNameKey::EncodeMinKeyForOrigin(origin_identifier_);
       const std::string stop_key =
-          DatabaseNameKey::EncodeStopKeyForOrigin(origin_identifier);
-      scoped_ptr<LevelDBIterator> it = db->CreateIterator();
+          DatabaseNameKey::EncodeStopKeyForOrigin(origin_identifier_);
+      scoped_ptr<LevelDBIterator> it = db_->CreateIterator();
       for (s = it->Seek(start_key);
            s.ok() && it->IsValid() && CompareKeys(it->Key(), stop_key) < 0;
            s = it->Next()) {
         int64 database_id = 0;
         found = false;
         s = GetInt(transaction.get(), it->Key(), &database_id, &found);
         if (!s.ok()) {
           INTERNAL_READ_ERROR_UNTESTED(SET_UP_METADATA);
           return false;
         }
         if (!found) {
           INTERNAL_CONSISTENCY_ERROR_UNTESTED(SET_UP_METADATA);
           return false;
         }
         std::string int_version_key = DatabaseMetaDataKey::Encode(
             database_id, DatabaseMetaDataKey::USER_INT_VERSION);
         PutVarInt(transaction.get(),
                   int_version_key,
                   IndexedDBDatabaseMetadata::DEFAULT_INT_VERSION);
       }
     }
     if (s.ok() && db_schema_version < 2) {
       db_schema_version = 2;
       PutInt(transaction.get(), schema_version_key, db_schema_version);
       db_data_version = blink::kSerializedScriptValueVersion;
       PutInt(transaction.get(), data_version_key, db_data_version);
     }
+    if (db_schema_version < 3) {
+      db_schema_version = 3;
+      if (!base::DeleteFile(blob_path_, true)) {
+        INTERNAL_WRITE_ERROR(SET_UP_METADATA);

[cmumford, 2014/05/06 18:33:11]

INTERNAL_READ_ERROR_UNTESTED

[ericu, 2014/05/06 20:07:42]

Same?

[cmumford, 2014/05/06 20:41:47]

Woops - my note was bad (READ vs. WRITE). But yes, we now only use the
*_UNTESTED version of the macros everywhere _unless_ we have a unit test that
hits that one line.

+        return false;
+      }
+    }
   }
 
   if (!s.ok()) {
     INTERNAL_READ_ERROR_UNTESTED(SET_UP_METADATA);
     return false;
   }
 
   // All new values will be written using this serialization version.
   found = false;
   s = GetInt(transaction.get(), data_version_key, &db_data_version, &found);
@@ skipping 56 matching lines @@
                                       scoped_ptr<LevelDBDatabase>* db,
                                       bool* is_disk_full) OVERRIDE {
     return LevelDBDatabase::Open(file_name, comparator, db, is_disk_full);
   }
   virtual leveldb::Status DestroyLevelDB(const base::FilePath& file_name)
       OVERRIDE {
     return LevelDBDatabase::Destroy(file_name);
   }
 };
 
+static bool GetBlobKeyGeneratorCurrentNumber(
+    LevelDBTransaction* leveldb_transaction,
+    int64 database_id,
+    int64* blob_key_generator_current_number) {
+  const std::string key_gen_key = DatabaseMetaDataKey::Encode(
+      database_id, DatabaseMetaDataKey::BLOB_KEY_GENERATOR_CURRENT_NUMBER);
+
+  // Default to initial number if not found.
+  int64 cur_number = DatabaseMetaDataKey::kBlobKeyGeneratorInitialNumber;
+  std::string data;
+
+  bool found = false;
+  bool ok = leveldb_transaction->Get(key_gen_key, &data, &found).ok();
+  if (!ok) {
+    INTERNAL_READ_ERROR(GET_BLOB_KEY_GENERATOR_CURRENT_NUMBER);

[cmumford, 2014/05/06 18:33:11]

INTERNAL_READ_ERROR_UNTESTED

[ericu, 2014/05/06 20:07:42]

Done.

+    return false;
+  }
+  if (found) {
+    StringPiece slice(data);
+    if (!DecodeVarInt(&slice, &cur_number) || !slice.empty() ||
+        !DatabaseMetaDataKey::IsValidBlobKey(cur_number)) {
+      INTERNAL_READ_ERROR(GET_BLOB_KEY_GENERATOR_CURRENT_NUMBER);

[cmumford, 2014/05/06 18:33:11]

INTERNAL_READ_ERROR_UNTESTED

[ericu, 2014/05/06 20:07:42]

Done.

+      return false;
+    }
+  }
+  *blob_key_generator_current_number = cur_number;
+  return true;
+}
+
+static bool UpdateBlobKeyGeneratorCurrentNumber(
+    LevelDBTransaction* leveldb_transaction,
+    int64 database_id,
+    int64 blob_key_generator_current_number) {
+#ifndef NDEBUG
+  int64 old_number;
+  if (!GetBlobKeyGeneratorCurrentNumber(
+          leveldb_transaction, database_id, &old_number))
+    return false;
+  DCHECK_LT(old_number, blob_key_generator_current_number);
+#endif
+  DCHECK(
+      DatabaseMetaDataKey::IsValidBlobKey(blob_key_generator_current_number));
+  const std::string key = DatabaseMetaDataKey::Encode(
+      database_id, DatabaseMetaDataKey::BLOB_KEY_GENERATOR_CURRENT_NUMBER);
+
+  PutVarInt(leveldb_transaction, key, blob_key_generator_current_number);
+  return true;
+}
+
 // TODO(ericu): Error recovery. If we persistently can't read the
 // blob journal, the safe thing to do is to clear it and leak the blobs,
 // though that may be costly. Still, database/directory deletion should always
 // clean things up, and we can write an fsck that will do a full correction if
 // need be.
 template <typename T>
 static leveldb::Status GetBlobJournal(const StringPiece& leveldb_key,
                                       T* leveldb_transaction,
                                       BlobJournalType* journal) {
   std::string data;
@@ skipping 73 matching lines @@
   if (!s.ok())
     return s;
   journal.push_back(
       std::make_pair(database_id, DatabaseMetaDataKey::kAllBlobsKey));
   std::string data;
   EncodeBlobJournal(journal, &data);
   leveldb_transaction->Put(key, &data);
   return leveldb::Status::OK();
 }
 
+// Blob Data is encoded as a series of:
+//   { is_file [bool], key [int64 as varInt],
+//     type [string-with-length, may be empty],
+//     (for Blobs only) size [int64 as varInt]
+//     (for Files only) fileName [string-with-length]
+//   }
+// There is no length field; just read until you run out of data.
+static std::string EncodeBlobData(
+    const std::vector<IndexedDBBlobInfo*>& blob_info) {
+  std::string ret;
+  std::vector<IndexedDBBlobInfo*>::const_iterator iter;
+  for (iter = blob_info.begin(); iter != blob_info.end(); ++iter) {
+    const IndexedDBBlobInfo& info = **iter;
+    EncodeBool(info.is_file(), &ret);
+    EncodeVarInt(info.key(), &ret);
+    EncodeStringWithLength(info.type(), &ret);
+    if (info.is_file())
+      EncodeStringWithLength(info.file_name(), &ret);
+    else
+      EncodeVarInt(info.size(), &ret);

[jsbell, 2014/05/06 18:29:48]

Can you add '(var int') to the docs at the top of indexed_db_leveldb_coding.cc ?

[ericu, 2014/05/06 20:07:42]

Done.  That's the blob journal, not this blob data, though.
Since this is done here, not in that file, it's not documented over there, only
in the comment above this function.

+  }
+  return ret;
+}
+
+static bool DecodeBlobData(const std::string& data,
+                           std::vector<IndexedDBBlobInfo>* output) {
+  std::vector<IndexedDBBlobInfo> ret;
+  output->clear();
+  StringPiece slice(data);
+  while (!slice.empty()) {
+    bool is_file;
+    int64 key;
+    base::string16 type;
+    int64 size;
+    base::string16 file_name;
+
+    if (!DecodeBool(&slice, &is_file))
+      return false;
+    if (!DecodeVarInt(&slice, &key) ||
+        !DatabaseMetaDataKey::IsValidBlobKey(key))
+      return false;
+    if (!DecodeStringWithLength(&slice, &type))
+      return false;
+    if (is_file) {
+      if (!DecodeStringWithLength(&slice, &file_name))
+        return false;
+      ret.push_back(IndexedDBBlobInfo(key, type, file_name));
+    } else {
+      if (!DecodeVarInt(&slice, &size) || size < 0)
+        return false;
+      ret.push_back(IndexedDBBlobInfo(type, static_cast<uint64>(size), key));
+    }
+  }
+  output->swap(ret);
+
+  return true;
+}
+
 IndexedDBBackingStore::IndexedDBBackingStore(
     IndexedDBFactory* indexed_db_factory,
     const GURL& origin_url,
     const base::FilePath& blob_path,
     net::URLRequestContext* request_context,
     scoped_ptr<LevelDBDatabase> db,
     scoped_ptr<LevelDBComparator> comparator,
     base::TaskRunner* task_runner)
     : indexed_db_factory_(indexed_db_factory),
       origin_url_(origin_url),
       blob_path_(blob_path),
       origin_identifier_(ComputeOriginIdentifier(origin_url)),
       request_context_(request_context),
       task_runner_(task_runner),
       db_(db.Pass()),
       comparator_(comparator.Pass()),
-      active_blob_registry_(this) {}
+      active_blob_registry_(this) {
+}
 
 IndexedDBBackingStore::~IndexedDBBackingStore() {
   if (!blob_path_.empty() && !child_process_ids_granted_.empty()) {
     ChildProcessSecurityPolicyImpl* policy =
         ChildProcessSecurityPolicyImpl::GetInstance();
     std::set<int>::const_iterator iter;
     for (iter = child_process_ids_granted_.begin();
          iter != child_process_ids_granted_.end();
          ++iter) {
       policy->RevokeAllPermissionsForFile(*iter, blob_path_);
@@ skipping 313 matching lines @@
                         origin_url);
   }
 
   if (!db) {
     NOTREACHED();
     HistogramOpenStatus(INDEXED_DB_BACKING_STORE_OPEN_FAILED_UNKNOWN_ERR,
                         origin_url);
     return scoped_refptr<IndexedDBBackingStore>();
   }
 
-  return Create(indexed_db_factory,
-                origin_url,
-                blob_path,
-                request_context,
-                db.Pass(),
-                comparator.Pass(),
-                task_runner);
+  scoped_refptr<IndexedDBBackingStore> backing_store =
+      Create(indexed_db_factory,
+             origin_url,
+             blob_path,
+             request_context,
+             db.Pass(),
+             comparator.Pass(),
+             task_runner);
+
+  if (clean_journal &&
+      !backing_store->CleanUpBlobJournal(LiveBlobJournalKey::Encode()).ok())

[cmumford, 2014/05/06 18:33:11]

If SetupMetaData fails for whatever reason backing_store will be NULL.

[ericu, 2014/05/06 20:07:42]

Fixed.

+    return scoped_refptr<IndexedDBBackingStore>();

[cmumford, 2014/05/06 18:33:11]

Do you want to record a histogram value here in case of failure?

[ericu, 2014/05/06 20:07:42]

Yeah, good idea.

+  return backing_store;
 }
 
 // static
 scoped_refptr<IndexedDBBackingStore> IndexedDBBackingStore::OpenInMemory(
     const GURL& origin_url,
     base::TaskRunner* task_runner) {
   DefaultLevelDBFactory leveldb_factory;
   return IndexedDBBackingStore::OpenInMemory(
       origin_url, &leveldb_factory, task_runner);
 }
@@ skipping 36 matching lines @@
     base::TaskRunner* task_runner) {
   // TODO(jsbell): Handle comparator name changes.
   scoped_refptr<IndexedDBBackingStore> backing_store(
       new IndexedDBBackingStore(indexed_db_factory,
                                 origin_url,
                                 blob_path,
                                 request_context,
                                 db.Pass(),
                                 comparator.Pass(),
                                 task_runner));
-  if (!SetUpMetadata(backing_store->db_.get(),
-                     backing_store->origin_identifier_))
+  if (!backing_store->SetUpMetadata())
     return scoped_refptr<IndexedDBBackingStore>();
 
   return backing_store;
 }
 
 void IndexedDBBackingStore::GrantChildProcessPermissions(int child_process_id) {
   if (!child_process_ids_granted_.count(child_process_id)) {
     child_process_ids_granted_.insert(child_process_id);
     ChildProcessSecurityPolicyImpl::GetInstance()->GrantReadFile(
         child_process_id, blob_path_);
@@ skipping 76 matching lines @@
 
   if (metadata->int_version == IndexedDBDatabaseMetadata::DEFAULT_INT_VERSION)
     metadata->int_version = IndexedDBDatabaseMetadata::NO_INT_VERSION;
 
   s = GetMaxObjectStoreId(
       db_.get(), metadata->id, &metadata->max_object_store_id);
   if (!s.ok()) {
     INTERNAL_READ_ERROR_UNTESTED(GET_IDBDATABASE_METADATA);
   }
 
+  // We don't cache this, we just check it if it's there.
+  int64 blob_key_generator_current_number =
+      DatabaseMetaDataKey::kInvalidBlobKey;
+
+  s = GetVarInt(
+      db_.get(),
+      DatabaseMetaDataKey::Encode(
+          metadata->id, DatabaseMetaDataKey::BLOB_KEY_GENERATOR_CURRENT_NUMBER),
+      &blob_key_generator_current_number,
+      found);
+  if (!s.ok()) {
+    INTERNAL_READ_ERROR(GET_IDBDATABASE_METADATA);

[cmumford, 2014/05/06 18:33:11]

_UNTESTED

[ericu, 2014/05/06 20:07:42]

Done.

+    return s;
+  }
+  if (!*found) {
+    // This database predates blob support.
+    *found = true;
+  } else if (!DatabaseMetaDataKey::IsValidBlobKey(
+                 blob_key_generator_current_number)) {
+    INTERNAL_CONSISTENCY_ERROR(GET_IDBDATABASE_METADATA);

[cmumford, 2014/05/06 18:33:11]

_UNTESTED

[ericu, 2014/05/06 20:07:42]

Done.

+    return InternalInconsistencyStatus();
+  }
+
   return s;
 }
 
 WARN_UNUSED_RESULT static leveldb::Status GetNewDatabaseId(
     LevelDBTransaction* transaction,
     int64* new_id) {
   *new_id = -1;
   int64 max_database_id = -1;
   bool found = false;
   leveldb::Status s =
@@ skipping 33 matching lines @@
          DatabaseNameKey::Encode(origin_identifier_, name),
          *row_id);
   PutString(
       transaction.get(),
       DatabaseMetaDataKey::Encode(*row_id, DatabaseMetaDataKey::USER_VERSION),
       version);
   PutVarInt(transaction.get(),
             DatabaseMetaDataKey::Encode(*row_id,
                                         DatabaseMetaDataKey::USER_INT_VERSION),
             int_version);
+  PutVarInt(
+      transaction.get(),
+      DatabaseMetaDataKey::Encode(
+          *row_id, DatabaseMetaDataKey::BLOB_KEY_GENERATOR_CURRENT_NUMBER),
+      DatabaseMetaDataKey::kBlobKeyGeneratorInitialNumber);
+
   s = transaction->Commit();
   if (!s.ok())
     INTERNAL_WRITE_ERROR_UNTESTED(CREATE_IDBDATABASE_METADATA);
   return s;
 }
 
 bool IndexedDBBackingStore::UpdateIDBDatabaseIntVersion(
     IndexedDBBackingStore::Transaction* transaction,
     int64 row_id,
     int64 int_version) {
@@ skipping 12 matching lines @@
                                    const std::string& end) {
   scoped_ptr<LevelDBIterator> it = transaction->CreateIterator();
   leveldb::Status s;
   for (s = it->Seek(begin);
        s.ok() && it->IsValid() && CompareKeys(it->Key(), end) < 0;
        s = it->Next())
     transaction->Remove(it->Key());
   return s;
 }
 
+static leveldb::Status DeleteBlobsInObjectStore(
+    IndexedDBBackingStore::Transaction* transaction,
+    int64 database_id,
+    int64 object_store_id) {
+  std::string start_key, end_key;
+  start_key =
+      BlobEntryKey::EncodeMinKeyForObjectStore(database_id, object_store_id);
+  end_key =
+      BlobEntryKey::EncodeStopKeyForObjectStore(database_id, object_store_id);
+
+  scoped_ptr<LevelDBIterator> it = transaction->transaction()->CreateIterator();
+  for (it->Seek(start_key);

[cmumford, 2014/05/06 18:33:11]

Seek/Next/etc. now return Status objects. should check these.

[ericu, 2014/05/06 20:07:42]

Working on that now.  On the last step of an iterator, to where it runs off the
end of the database, does it still return s.ok() even if !it->IsValid()?  I'm
crawling through the leveldb code, and it's not clear [and is AFAICT
undocumented].

[cmumford, 2014/05/06 20:41:47]

I believe that it will return OK() and the iterator will be !valid, but I'm not
100% on that. When I modified those routines to return status codes I left in
the IsValid() checks and added OK() checks to them and all tests still pass
(FWTW).

+       it->IsValid() && CompareKeys(it->Key(), end_key) < 0;
+       it->Next()) {
+    StringPiece key_piece(it->Key());
+    std::string user_key =
+        BlobEntryKey::ReencodeToObjectStoreDataKey(&key_piece);
+    if (!user_key.size()) {
+      INTERNAL_CONSISTENCY_ERROR(GET_IDBDATABASE_METADATA);

[cmumford, 2014/05/06 18:33:11]

_UNTESTED

[ericu, 2014/05/06 20:07:42]

Done.

+      return InternalInconsistencyStatus();
+    }
+    transaction->PutBlobInfo(
+        database_id, object_store_id, user_key, NULL, NULL);
+  }
+  return leveldb::Status::OK();
+}
+
 leveldb::Status IndexedDBBackingStore::DeleteDatabase(
     const base::string16& name) {
   IDB_TRACE("IndexedDBBackingStore::DeleteDatabase");
   scoped_ptr<LevelDBDirectTransaction> transaction =
       LevelDBDirectTransaction::Create(db_.get());
 
+  leveldb::Status s;
+  s = CleanUpBlobJournal(BlobJournalKey::Encode());
+  if (!s.ok())
+    return InternalInconsistencyStatus();

[cmumford, 2014/05/06 18:33:11]

Why not return "s" here?

[ericu, 2014/05/06 20:07:42]

Done.

+
   IndexedDBDatabaseMetadata metadata;
   bool success = false;
-  leveldb::Status s = GetIDBDatabaseMetaData(name, &metadata, &success);
+  s = GetIDBDatabaseMetaData(name, &metadata, &success);
   if (!s.ok())
     return s;
   if (!success)
     return leveldb::Status::OK();
 
   const std::string start_key = DatabaseMetaDataKey::Encode(
       metadata.id, DatabaseMetaDataKey::ORIGIN_NAME);
   const std::string stop_key = DatabaseMetaDataKey::Encode(
       metadata.id + 1, DatabaseMetaDataKey::ORIGIN_NAME);
   scoped_ptr<LevelDBIterator> it = db_->CreateIterator();
   for (s = it->Seek(start_key);
        s.ok() && it->IsValid() && CompareKeys(it->Key(), stop_key) < 0;
        s = it->Next())
     transaction->Remove(it->Key());
   if (!s.ok()) {
     INTERNAL_WRITE_ERROR_UNTESTED(DELETE_DATABASE);
     return s;
   }
 
   const std::string key = DatabaseNameKey::Encode(origin_identifier_, name);
   transaction->Remove(key);
 
-  // TODO(ericu): Put the real calls to the blob journal code here.  For now,
-  // I've inserted fake calls so that we don't get "you didn't use this static
-  // function" compiler errors.
+  bool need_cleanup = false;
+  if (active_blob_registry()->MarkDeletedCheckIfUsed(
+          metadata.id, DatabaseMetaDataKey::kAllBlobsKey)) {
+    s = MergeDatabaseIntoLiveBlobJournal(transaction.get(), metadata.id);
+    if (!s.ok())
+      return s;
+  } else {
+    UpdateBlobJournalWithDatabase(transaction.get(), metadata.id);
+    need_cleanup = true;
+  }
+
+  // TODO(ericu): Remove these fake calls, added to avoid "defined but unused"
+  // compiler errors until the code that makes the real calls can be added.
   if (false) {
+    std::vector<IndexedDBBlobInfo*> fake;
+    EncodeBlobData(fake);
+
     scoped_refptr<LevelDBTransaction> fake_transaction =
         new LevelDBTransaction(NULL);
     BlobJournalType fake_journal;
-    MergeDatabaseIntoLiveBlobJournal(transaction.get(), metadata.id);
-    UpdateBlobJournalWithDatabase(transaction.get(), metadata.id);
     MergeBlobsIntoLiveBlobJournal(fake_transaction.get(), fake_journal);
+    UpdateBlobKeyGeneratorCurrentNumber(fake_transaction.get(), 0, 0);
   }
 
   s = transaction->Commit();
   if (!s.ok()) {
     INTERNAL_WRITE_ERROR_UNTESTED(DELETE_DATABASE);
     return s;
   }
+
+  if (need_cleanup)
+    CleanUpBlobJournal(BlobJournalKey::Encode());
+
   db_->Compact(start_key, stop_key);
   return s;
 }
 
 static bool CheckObjectStoreAndMetaDataType(const LevelDBIterator* it,
                                             const std::string& stop_key,
                                             int64 object_store_id,
                                             int64 meta_data_type) {
   if (!it->IsValid() || CompareKeys(it->Key(), stop_key) >= 0)
     return false;
@@ skipping 288 matching lines @@
                 &found);
   if (!s.ok()) {
     INTERNAL_READ_ERROR_UNTESTED(DELETE_OBJECT_STORE);
     return s;
   }
   if (!found) {
     INTERNAL_CONSISTENCY_ERROR_UNTESTED(DELETE_OBJECT_STORE);
     return InternalInconsistencyStatus();
   }
 
+  s = DeleteBlobsInObjectStore(transaction, database_id, object_store_id);
+  if (!s.ok()) {
+    INTERNAL_CONSISTENCY_ERROR(DELETE_OBJECT_STORE);

[cmumford, 2014/05/06 18:33:11]

_UNTESTED

[ericu, 2014/05/06 20:07:42]

Done.

+    return s;
+  }
+
   s = DeleteRange(
       leveldb_transaction,
       ObjectStoreMetaDataKey::Encode(database_id, object_store_id, 0),
       ObjectStoreMetaDataKey::EncodeMaxKey(database_id, object_store_id));
 
   if (s.ok()) {
     leveldb_transaction->Remove(
         ObjectStoreNamesKey::Encode(database_id, object_store_name));
 
     s = DeleteRange(
@@ skipping 48 matching lines @@
   }
 
   int64 version;
   StringPiece slice(data);
   if (!DecodeVarInt(&slice, &version)) {
     INTERNAL_READ_ERROR_UNTESTED(GET_RECORD);
     return InternalInconsistencyStatus();
   }
 
   record->bits = slice.as_string();
-  return s;
+  return transaction->GetBlobInfoForRecord(database_id, leveldb_key, record);
 }
 
 WARN_UNUSED_RESULT static leveldb::Status GetNewVersionNumber(
     LevelDBTransaction* transaction,
     int64 database_id,
     int64 object_store_id,
     int64* new_version_number) {
   const std::string last_version_key = ObjectStoreMetaDataKey::Encode(
       database_id, object_store_id, ObjectStoreMetaDataKey::LAST_VERSION);
 
@@ skipping 75 matching lines @@
   IDB_TRACE("IndexedDBBackingStore::ClearObjectStore");
   if (!KeyPrefix::ValidIds(database_id, object_store_id))
     return InvalidDBKeyStatus();
   const std::string start_key =
       KeyPrefix(database_id, object_store_id).Encode();
   const std::string stop_key =
       KeyPrefix(database_id, object_store_id + 1).Encode();
 
   leveldb::Status s =
       DeleteRange(transaction->transaction(), start_key, stop_key);
-  if (!s.ok())
+  if (!s.ok()) {
     INTERNAL_WRITE_ERROR(CLEAR_OBJECT_STORE);
-  return s;
+    return s;
+  }
+  return DeleteBlobsInObjectStore(transaction, database_id, object_store_id);
 }
 
 leveldb::Status IndexedDBBackingStore::DeleteRecord(
     IndexedDBBackingStore::Transaction* transaction,
     int64 database_id,
     int64 object_store_id,
     const RecordIdentifier& record_identifier) {
   IDB_TRACE("IndexedDBBackingStore::DeleteRecord");
   if (!KeyPrefix::ValidIds(database_id, object_store_id))
     return InvalidDBKeyStatus();
@@ skipping 577 matching lines @@
     } else {
       DCHECK(DatabaseMetaDataKey::IsValidBlobKey(blob_key));
       if (!RemoveBlobFile(database_id, blob_key))
         return IOErrorStatus();
     }
   }
   ClearBlobJournal(journal_transaction.get(), level_db_key);
   return journal_transaction->Commit();
 }
 
+leveldb::Status IndexedDBBackingStore::Transaction::GetBlobInfoForRecord(
+    int64 database_id,
+    const std::string& object_store_data_key,
+    IndexedDBValue* value) {
+  BlobChangeRecord* change_record = NULL;
+  BlobChangeMap::const_iterator blob_iter =
+      blob_change_map_.find(object_store_data_key);
+  if (blob_iter != blob_change_map_.end())
+    change_record = blob_iter->second;
+  else {

[jsbell, 2014/05/06 18:29:48]

Inconsistent bracing (if any if/else branch uses braces, all must)

[ericu, 2014/05/06 20:07:42]

Done.

+    blob_iter = incognito_blob_map_.find(object_store_data_key);
+    if (blob_iter != incognito_blob_map_.end())
+      change_record = blob_iter->second;
+  }
+  if (change_record) {
+    // Either we haven't written the blob to disk yet or we're in incognito
+    // mode, so we have to send back the one they sent us.  This change record
+    // includes the original UUID.
+    value->blob_info = change_record->blob_info();
+    return leveldb::Status::OK();
+  }
+
+  BlobEntryKey blob_entry_key;
+  StringPiece leveldb_key_piece(object_store_data_key);
+  if (!BlobEntryKey::FromObjectStoreDataKey(&leveldb_key_piece,
+                                            &blob_entry_key)) {
+    NOTREACHED();
+    return InternalInconsistencyStatus();
+  }
+  scoped_ptr<LevelDBIterator> it = transaction()->CreateIterator();
+  std::string encoded_key = blob_entry_key.Encode();
+  it->Seek(encoded_key);

[cmumford, 2014/05/06 18:33:11]

Status

[ericu, 2014/05/06 20:07:42]

Done.

+  if (it->IsValid() && CompareKeys(it->Key(), encoded_key) == 0) {
+    if (!DecodeBlobData(it->Value().as_string(), &value->blob_info)) {
+      INTERNAL_READ_ERROR(GET_BLOB_INFO_FOR_RECORD);
+      return InternalInconsistencyStatus();
+    }
+    std::vector<IndexedDBBlobInfo>::iterator iter;
+    for (iter = value->blob_info.begin(); iter != value->blob_info.end();
+         ++iter) {
+      iter->set_file_path(
+          backing_store_->GetBlobFileName(database_id, iter->key()));
+      iter->set_mark_used_callback(
+          backing_store_->active_blob_registry()->GetAddBlobRefCallback(
+              database_id, iter->key()));
+      iter->set_release_callback(
+          backing_store_->active_blob_registry()->GetFinalReleaseCallback(
+              database_id, iter->key()));
+      if (iter->is_file()) {
+        base::File::Info info;
+        if (base::GetFileInfo(iter->file_path(), &info)) {
+          // This should always work, but it isn't fatal if it doesn't; it just
+          // means a potential slow synchronous call from the renderer later.
+          iter->set_last_modified(info.last_modified);
+          iter->set_size(info.size);
+        }
+      }
+    }
+  }
+  return leveldb::Status::OK();
+}
+
 void IndexedDBBackingStore::CleanPrimaryJournalIgnoreReturn() {
   CleanUpBlobJournal(BlobJournalKey::Encode());
 }
 
 WARN_UNUSED_RESULT static leveldb::Status SetMaxIndexId(
     LevelDBTransaction* transaction,
     int64 database_id,
     int64 object_store_id,
     int64 index_id) {
   int64 max_index_id = -1;
@@ skipping 303 matching lines @@
 
   StringPiece slice(found_encoded_primary_key);
   if (DecodeIDBKey(&slice, found_primary_key) && slice.empty())
     return s;
   else
     return InvalidDBKeyStatus();
 }
 
 IndexedDBBackingStore::Cursor::Cursor(
     const IndexedDBBackingStore::Cursor* other)
-    : transaction_(other->transaction_),
+    : backing_store_(other->backing_store_),
+      transaction_(other->transaction_),
+      database_id_(other->database_id_),
       cursor_options_(other->cursor_options_),
       current_key_(new IndexedDBKey(*other->current_key_)) {
   if (other->iterator_) {
-    iterator_ = transaction_->CreateIterator();
+    iterator_ = transaction_->transaction()->CreateIterator();
 
     if (other->iterator_->IsValid()) {
       leveldb::Status s = iterator_->Seek(other->iterator_->Key());
       // TODO(cmumford): Handle this error (crbug.com/363397)
       DCHECK(iterator_->IsValid());
     }
   }
 }
 
-IndexedDBBackingStore::Cursor::Cursor(LevelDBTransaction* transaction,
-                                      const CursorOptions& cursor_options)
-    : transaction_(transaction), cursor_options_(cursor_options) {}
+IndexedDBBackingStore::Cursor::Cursor(
+    scoped_refptr<IndexedDBBackingStore> backing_store,
+    IndexedDBBackingStore::Transaction* transaction,
+    int64 database_id,
+    const CursorOptions& cursor_options)
+    : backing_store_(backing_store),
+      transaction_(transaction),
+      database_id_(database_id),
+      cursor_options_(cursor_options) {
+}
 IndexedDBBackingStore::Cursor::~Cursor() {}
 
 bool IndexedDBBackingStore::Cursor::FirstSeek(leveldb::Status* s) {
-  iterator_ = transaction_->CreateIterator();
+  iterator_ = transaction_->transaction()->CreateIterator();
   if (cursor_options_.forward)
     *s = iterator_->Seek(cursor_options_.low_key);
   else
     *s = iterator_->Seek(cursor_options_.high_key);
   if (!s->ok())
     return false;
 
   return Continue(0, READY, s);
 }
 
@@ skipping 162 matching lines @@
 }
 
 const IndexedDBBackingStore::RecordIdentifier&
 IndexedDBBackingStore::Cursor::record_identifier() const {
   return record_identifier_;
 }
 
 class ObjectStoreKeyCursorImpl : public IndexedDBBackingStore::Cursor {
  public:
   ObjectStoreKeyCursorImpl(
-      LevelDBTransaction* transaction,
+      scoped_refptr<IndexedDBBackingStore> backing_store,
+      IndexedDBBackingStore::Transaction* transaction,
+      int64 database_id,
       const IndexedDBBackingStore::Cursor::CursorOptions& cursor_options)
-      : IndexedDBBackingStore::Cursor(transaction, cursor_options) {}
+      : IndexedDBBackingStore::Cursor(backing_store,
+                                      transaction,
+                                      database_id,
+                                      cursor_options) {}
 
   virtual Cursor* Clone() OVERRIDE {
     return new ObjectStoreKeyCursorImpl(this);
   }
 
   // IndexedDBBackingStore::Cursor
   virtual IndexedDBValue* value() OVERRIDE {
     NOTREACHED();
     return NULL;
   }
@@ skipping 36 matching lines @@
   std::string encoded_key;
   EncodeIDBKey(*current_key_, &encoded_key);
   record_identifier_.Reset(encoded_key, version);
 
   return true;
 }
 
 class ObjectStoreCursorImpl : public IndexedDBBackingStore::Cursor {
  public:
   ObjectStoreCursorImpl(
-      LevelDBTransaction* transaction,
+      scoped_refptr<IndexedDBBackingStore> backing_store,
+      IndexedDBBackingStore::Transaction* transaction,
+      int64 database_id,
       const IndexedDBBackingStore::Cursor::CursorOptions& cursor_options)
-      : IndexedDBBackingStore::Cursor(transaction, cursor_options) {}
+      : IndexedDBBackingStore::Cursor(backing_store,
+                                      transaction,
+                                      database_id,
+                                      cursor_options) {}
 
   virtual Cursor* Clone() OVERRIDE { return new ObjectStoreCursorImpl(this); }
 
   // IndexedDBBackingStore::Cursor
   virtual IndexedDBValue* value() OVERRIDE { return &current_value_; }
   virtual bool LoadCurrentRow() OVERRIDE;
 
  protected:
   virtual std::string EncodeKey(const IndexedDBKey& key) OVERRIDE {
     return ObjectStoreDataKey::Encode(
@@ skipping 28 matching lines @@
   if (!DecodeVarInt(&value_slice, &version)) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
   // TODO(jsbell): This re-encodes what was just decoded; try and optimize.
   std::string encoded_key;
   EncodeIDBKey(*current_key_, &encoded_key);
   record_identifier_.Reset(encoded_key, version);
 
+  if (!transaction_->GetBlobInfoForRecord(database_id_,
+                                          iterator_->Key().as_string(),
+                                          &current_value_).ok()) {
+    return false;
+  }
   current_value_.bits = value_slice.as_string();
   return true;
 }
 
 class IndexKeyCursorImpl : public IndexedDBBackingStore::Cursor {
  public:
   IndexKeyCursorImpl(
-      LevelDBTransaction* transaction,
+      scoped_refptr<IndexedDBBackingStore> backing_store,
+      IndexedDBBackingStore::Transaction* transaction,
+      int64 database_id,
       const IndexedDBBackingStore::Cursor::CursorOptions& cursor_options)
-      : IndexedDBBackingStore::Cursor(transaction, cursor_options) {}
+      : IndexedDBBackingStore::Cursor(backing_store,
+                                      transaction,
+                                      database_id,
+                                      cursor_options) {}
 
   virtual Cursor* Clone() OVERRIDE { return new IndexKeyCursorImpl(this); }
 
   // IndexedDBBackingStore::Cursor
   virtual IndexedDBValue* value() OVERRIDE {
     NOTREACHED();
     return NULL;
   }
   virtual const IndexedDBKey& primary_key() const OVERRIDE {
     return *primary_key_;
@@ skipping 52 matching lines @@
     return false;
   }
 
   std::string primary_leveldb_key =
       ObjectStoreDataKey::Encode(index_data_key.DatabaseId(),
                                  index_data_key.ObjectStoreId(),
                                  *primary_key_);
 
   std::string result;
   bool found = false;
-  leveldb::Status s = transaction_->Get(primary_leveldb_key, &result, &found);
+  leveldb::Status s =
+      transaction_->transaction()->Get(primary_leveldb_key, &result, &found);
   if (!s.ok()) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
   if (!found) {
-    transaction_->Remove(iterator_->Key());
+    transaction_->transaction()->Remove(iterator_->Key());
     return false;
   }
   if (!result.size()) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
   int64 object_store_data_version;
   slice = StringPiece(result);
   if (!DecodeVarInt(&slice, &object_store_data_version)) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
   if (object_store_data_version != index_data_version) {
-    transaction_->Remove(iterator_->Key());
+    transaction_->transaction()->Remove(iterator_->Key());
     return false;
   }
 
   return true;
 }
 
 class IndexCursorImpl : public IndexedDBBackingStore::Cursor {
  public:
   IndexCursorImpl(
-      LevelDBTransaction* transaction,
+      scoped_refptr<IndexedDBBackingStore> backing_store,
+      IndexedDBBackingStore::Transaction* transaction,
+      int64 database_id,
       const IndexedDBBackingStore::Cursor::CursorOptions& cursor_options)
-      : IndexedDBBackingStore::Cursor(transaction, cursor_options) {}
+      : IndexedDBBackingStore::Cursor(backing_store,
+                                      transaction,
+                                      database_id,
+                                      cursor_options) {}
 
   virtual Cursor* Clone() OVERRIDE { return new IndexCursorImpl(this); }
 
   // IndexedDBBackingStore::Cursor
   virtual IndexedDBValue* value() OVERRIDE { return &current_value_; }
   virtual const IndexedDBKey& primary_key() const OVERRIDE {
     return *primary_key_;
   }
   virtual const IndexedDBBackingStore::RecordIdentifier& record_identifier()
       const OVERRIDE {
@@ skipping 45 matching lines @@
   int64 index_data_version;
   if (!DecodeVarInt(&slice, &index_data_version)) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
   if (!DecodeIDBKey(&slice, &primary_key_)) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
+  DCHECK_EQ(index_data_key.DatabaseId(), database_id_);
   primary_leveldb_key_ =
       ObjectStoreDataKey::Encode(index_data_key.DatabaseId(),
                                  index_data_key.ObjectStoreId(),
                                  *primary_key_);
 
   std::string result;
   bool found = false;
-  leveldb::Status s = transaction_->Get(primary_leveldb_key_, &result, &found);
+  leveldb::Status s =
+      transaction_->transaction()->Get(primary_leveldb_key_, &result, &found);
   if (!s.ok()) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
   if (!found) {
-    transaction_->Remove(iterator_->Key());
+    transaction_->transaction()->Remove(iterator_->Key());
     return false;
   }
   if (!result.size()) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
   int64 object_store_data_version;
   slice = StringPiece(result);
   if (!DecodeVarInt(&slice, &object_store_data_version)) {
     INTERNAL_READ_ERROR_UNTESTED(LOAD_CURRENT_ROW);
     return false;
   }
 
   if (object_store_data_version != index_data_version) {
-    transaction_->Remove(iterator_->Key());
+    transaction_->transaction()->Remove(iterator_->Key());
     return false;
   }
 
   current_value_.bits = slice.as_string();
-  return true;
+  return transaction_->GetBlobInfoForRecord(database_id_,
+                                            primary_leveldb_key_,
+                                            &current_value_).ok();
 }
 
 bool ObjectStoreCursorOptions(
     LevelDBTransaction* transaction,
     int64 database_id,
     int64 object_store_id,
     const IndexedDBKeyRange& range,
     indexed_db::CursorDirection direction,
     IndexedDBBackingStore::Cursor::CursorOptions* cursor_options) {
   cursor_options->database_id = database_id;
@@ skipping 146 matching lines @@
   *s = leveldb::Status::OK();
   LevelDBTransaction* leveldb_transaction = transaction->transaction();
   IndexedDBBackingStore::Cursor::CursorOptions cursor_options;
   if (!ObjectStoreCursorOptions(leveldb_transaction,
                                 database_id,
                                 object_store_id,
                                 range,
                                 direction,
                                 &cursor_options))
     return scoped_ptr<IndexedDBBackingStore::Cursor>();
-  scoped_ptr<ObjectStoreCursorImpl> cursor(
-      new ObjectStoreCursorImpl(leveldb_transaction, cursor_options));
+  scoped_ptr<ObjectStoreCursorImpl> cursor(new ObjectStoreCursorImpl(
+      this, transaction, database_id, cursor_options));
   if (!cursor->FirstSeek(s))
     return scoped_ptr<IndexedDBBackingStore::Cursor>();
 
   return cursor.PassAs<IndexedDBBackingStore::Cursor>();
 }
 
 scoped_ptr<IndexedDBBackingStore::Cursor>
 IndexedDBBackingStore::OpenObjectStoreKeyCursor(
     IndexedDBBackingStore::Transaction* transaction,
     int64 database_id,
     int64 object_store_id,
     const IndexedDBKeyRange& range,
     indexed_db::CursorDirection direction,
     leveldb::Status* s) {
   IDB_TRACE("IndexedDBBackingStore::OpenObjectStoreKeyCursor");
   *s = leveldb::Status::OK();
   LevelDBTransaction* leveldb_transaction = transaction->transaction();
   IndexedDBBackingStore::Cursor::CursorOptions cursor_options;
   if (!ObjectStoreCursorOptions(leveldb_transaction,
                                 database_id,
                                 object_store_id,
                                 range,
                                 direction,
                                 &cursor_options))
     return scoped_ptr<IndexedDBBackingStore::Cursor>();
-  scoped_ptr<ObjectStoreKeyCursorImpl> cursor(
-      new ObjectStoreKeyCursorImpl(leveldb_transaction, cursor_options));
+  scoped_ptr<ObjectStoreKeyCursorImpl> cursor(new ObjectStoreKeyCursorImpl(
+      this, transaction, database_id, cursor_options));
   if (!cursor->FirstSeek(s))
     return scoped_ptr<IndexedDBBackingStore::Cursor>();
 
   return cursor.PassAs<IndexedDBBackingStore::Cursor>();
 }
 
 scoped_ptr<IndexedDBBackingStore::Cursor>
 IndexedDBBackingStore::OpenIndexKeyCursor(
     IndexedDBBackingStore::Transaction* transaction,
     int64 database_id,
     int64 object_store_id,
     int64 index_id,
     const IndexedDBKeyRange& range,
     indexed_db::CursorDirection direction,
     leveldb::Status* s) {
   IDB_TRACE("IndexedDBBackingStore::OpenIndexKeyCursor");
   *s = leveldb::Status::OK();
   LevelDBTransaction* leveldb_transaction = transaction->transaction();
   IndexedDBBackingStore::Cursor::CursorOptions cursor_options;
   if (!IndexCursorOptions(leveldb_transaction,
                           database_id,
                           object_store_id,
                           index_id,
                           range,
                           direction,
                           &cursor_options))
     return scoped_ptr<IndexedDBBackingStore::Cursor>();
   scoped_ptr<IndexKeyCursorImpl> cursor(
-      new IndexKeyCursorImpl(leveldb_transaction, cursor_options));
+      new IndexKeyCursorImpl(this, transaction, database_id, cursor_options));
   if (!cursor->FirstSeek(s))
     return scoped_ptr<IndexedDBBackingStore::Cursor>();
 
   return cursor.PassAs<IndexedDBBackingStore::Cursor>();
 }
 
 scoped_ptr<IndexedDBBackingStore::Cursor>
 IndexedDBBackingStore::OpenIndexCursor(
     IndexedDBBackingStore::Transaction* transaction,
     int64 database_id,
     int64 object_store_id,
     int64 index_id,
     const IndexedDBKeyRange& range,
     indexed_db::CursorDirection direction,
     leveldb::Status* s) {
   IDB_TRACE("IndexedDBBackingStore::OpenIndexCursor");
   LevelDBTransaction* leveldb_transaction = transaction->transaction();
   IndexedDBBackingStore::Cursor::CursorOptions cursor_options;
   if (!IndexCursorOptions(leveldb_transaction,
                           database_id,
                           object_store_id,
                           index_id,
                           range,
                           direction,
                           &cursor_options))
     return scoped_ptr<IndexedDBBackingStore::Cursor>();
   scoped_ptr<IndexCursorImpl> cursor(
-      new IndexCursorImpl(leveldb_transaction, cursor_options));
+      new IndexCursorImpl(this, transaction, database_id, cursor_options));
   if (!cursor->FirstSeek(s))
     return scoped_ptr<IndexedDBBackingStore::Cursor>();
 
   return cursor.PassAs<IndexedDBBackingStore::Cursor>();
 }
 
 IndexedDBBackingStore::Transaction::Transaction(
     IndexedDBBackingStore* backing_store)
     : backing_store_(backing_store), database_id_(-1) {
 }
@@ skipping 171 matching lines @@
     const GURL& url,
     int64_t key)
     : is_file_(false), url_(url), key_(key) {}
 
 IndexedDBBackingStore::Transaction::WriteDescriptor::WriteDescriptor(
     const FilePath& file_path,
     int64_t key)
     : is_file_(true), file_path_(file_path), key_(key) {}
 
 }  // namespace content