Blob support for IDB [Chromium]

content/browser/indexed_db/indexed_db_leveldb_coding.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/indexed_db/indexed_db_leveldb_coding.h"
 
 #include <iterator>
 #include <limits>
 
 #include "base/logging.h"
@@ skipping 26 matching lines @@
 // Strings (origins, names, etc) are encoded as UTF-16BE.
 //
 //
 // Global metadata
 // ---------------
 // The prefix is <0, 0, 0>, followed by a metadata type byte:
 //
 // <0, 0, 0, 0> => backing store schema version [SchemaVersionKey]
 // <0, 0, 0, 1> => maximum allocated database [MaxDatabaseIdKey]
 // <0, 0, 0, 2> => SerializedScriptValue version [DataVersionKey]
+// <0, 0, 0, 3>
+//   => Blob journal
+//     The format of the journal is: {database_id, blobKey}*.
+//     If the blobKey is AllBlobsKey, the whole database should be deleted.
+//     [BlobJournalKey]
+// <0, 0, 0, 4> => Live blob journal; same format. [LiveBlobJournalKey]
 // <0, 0, 0, 100, database id>
 //   => Existence implies the database id is in the free list
 //      [DatabaseFreeListKey]
 // <0, 0, 0, 201, origin, database name> => Database id [DatabaseNameKey]
 //
 //
 // Database metadata: [DatabaseMetaDataKey]
 // ----------------------------------------
 // The prefix is <database id, 0, 0> followed by a metadata type byte:
 //
 // <database id, 0, 0, 0> => origin name
 // <database id, 0, 0, 1> => database name
 // <database id, 0, 0, 2> => IDB string version data (obsolete)
 // <database id, 0, 0, 3> => maximum allocated object store id
 // <database id, 0, 0, 4> => IDB integer version (var int)
+// <database id, 0, 0, 5> => blob key generator current number
 //
 //
 // Object store metadata: [ObjectStoreMetaDataKey]
 // -----------------------------------------------
 // The prefix is <database id, 0, 0>, followed by a type byte (50), then the
 // object store id (var int), then a metadata type byte.
 //
 // <database id, 0, 0, 50, object store id, 0> => object store name
 // <database id, 0, 0, 50, object store id, 1> => key path
 // <database id, 0, 0, 50, object store id, 2> => auto increment flag
@@ skipping 54 matching lines @@
 //   => "version", serialized script value
 //
 //
 // "Exists" entry: [ExistsEntryKey]
 // --------------------------------
 // The prefix is followed by a type byte and the encoded IDB primary key.
 //
 // <database id, object store id, 2, user key> => "version"
 //
 //
+// Blob entry table: [BlobEntryKey]
+// --------------------------------
+//
+// The prefix is followed by a type byte and the encoded IDB primary key.
+//
+// <database id, object store id, 3, user key> => array of IndexedDBBlobInfo
+//
+//
 // Index data
 // ----------
 // The prefix is followed by a type byte, the encoded IDB index key, a
 // "sequence" number (obsolete; var int), and the encoded IDB primary key.
 //
 // <database id, object store id, index id, index key, sequence number,
 //   primary key> => "version", primary key [IndexDataKey]
 //
 // The sequence number is obsolete; it was used to allow two entries with the
 // same user (index) key in non-unique indexes prior to the inclusion of the
@@ skipping 10 matching lines @@
 using WebKit::WebIDBKeyTypeNumber;
 using WebKit::WebIDBKeyTypeString;
 using WebKit::WebIDBKeyPathType;
 using WebKit::WebIDBKeyPathTypeArray;
 using WebKit::WebIDBKeyPathTypeNull;
 using WebKit::WebIDBKeyPathTypeString;
 
 namespace content {
 
 // As most of the IndexedDBKeys and encoded values are short, we
-// initialize some Vectors with a default inline buffer size to reduce
-// the memory re-allocations when the Vectors are appended.
+// initialize some std::vectors with a default inline buffer size to reduce
+// the memory re-allocations when the std::vectors are appended.
 static const size_t kDefaultInlineBufferSize = 32;
 
 static const unsigned char kIndexedDBKeyNullTypeByte = 0;
 static const unsigned char kIndexedDBKeyStringTypeByte = 1;
 static const unsigned char kIndexedDBKeyDateTypeByte = 2;
 static const unsigned char kIndexedDBKeyNumberTypeByte = 3;
 static const unsigned char kIndexedDBKeyArrayTypeByte = 4;
 static const unsigned char kIndexedDBKeyMinKeyTypeByte = 5;
 
 static const unsigned char kIndexedDBKeyPathTypeCodedByte1 = 0;
 static const unsigned char kIndexedDBKeyPathTypeCodedByte2 = 0;
 
 static const unsigned char kObjectStoreDataIndexId = 1;
 static const unsigned char kExistsEntryIndexId = 2;
+static const unsigned char kBlobEntryIndexId = 3;
 
 static const unsigned char kSchemaVersionTypeByte = 0;
 static const unsigned char kMaxDatabaseIdTypeByte = 1;
 static const unsigned char kDataVersionTypeByte = 2;
+static const unsigned char kBlobJournalTypeByte = 3;
+static const unsigned char kLiveBlobJournalTypeByte = 4;
 static const unsigned char kMaxSimpleGlobalMetaDataTypeByte =
-    3;  // Insert before this and increment.
+    5;  // Insert before this and increment.
 static const unsigned char kDatabaseFreeListTypeByte = 100;
 static const unsigned char kDatabaseNameTypeByte = 201;
 
 static const unsigned char kObjectStoreMetaDataTypeByte = 50;
 static const unsigned char kIndexMetaDataTypeByte = 100;
 static const unsigned char kObjectStoreFreeListTypeByte = 150;
 static const unsigned char kIndexFreeListTypeByte = 151;
 static const unsigned char kObjectStoreNamesTypeByte = 200;
 static const unsigned char kIndexNamesKeyTypeByte = 201;
 
@@ skipping 653 matching lines @@
   DCHECK(!slice_a.empty());
   DCHECK(!slice_b.empty());
   // Prefixes are not compared - it is assumed this was already done.
   DCHECK(!prefix_a.Compare(prefix_b));
 
   return CompareSuffix<ObjectStoreDataKey>(
       &slice_a, &slice_b, only_compare_index_keys, ok);
 }
 
 template <>
+int Compare<BlobEntryKey>(const StringPiece& a,
+                          const StringPiece& b,
+                          bool,
+                          bool* ok) {
+  KeyPrefix prefix_a;
+  KeyPrefix prefix_b;
+  StringPiece slice_a(a);
+  StringPiece slice_b(b);
+  bool ok_a = KeyPrefix::Decode(&slice_a, &prefix_a);
+  bool ok_b = KeyPrefix::Decode(&slice_b, &prefix_b);
+  DCHECK(ok_a);
+  DCHECK(ok_b);
+  DCHECK(prefix_a.database_id_);
+  DCHECK(prefix_a.object_store_id_);
+  DCHECK_EQ(prefix_a.index_id_, BlobEntryKey::kSpecialIndexNumber);
+  DCHECK(prefix_b.database_id_);
+  DCHECK(prefix_b.object_store_id_);
+  DCHECK_EQ(prefix_b.index_id_, BlobEntryKey::kSpecialIndexNumber);
+  DCHECK(!slice_a.empty());
+  DCHECK(!slice_b.empty());
+  // Prefixes are not compared - it is assumed this was already done.
+  DCHECK(!prefix_a.Compare(prefix_b));
+
+  return CompareSuffix<ObjectStoreDataKey>(
+      &slice_a, &slice_b, false, ok);
+}
+
+template <>
 int CompareSuffix<IndexDataKey>(StringPiece* slice_a,
                                 StringPiece* slice_b,
                                 bool only_compare_index_keys,
                                 bool* ok) {
   // index key
   int result = CompareEncodedIDBKeys(slice_a, slice_b, ok);
   if (!*ok || result)
     return result;
   if (only_compare_index_keys)
     return 0;
@@ skipping 158 matching lines @@
 
     case KeyPrefix::EXISTS_ENTRY: {
       // Provide a stable ordering for invalid data.
       if (slice_a.empty() || slice_b.empty())
         return CompareSizes(slice_a.size(), slice_b.size());
 
       return CompareSuffix<ExistsEntryKey>(
           &slice_a, &slice_b, /*only_compare_index_keys*/ false, ok);
     }
 
+    case KeyPrefix::BLOB_ENTRY: {
+      // Provide a stable ordering for invalid data.
+      if (slice_a.empty() || slice_b.empty())
+        return CompareSizes(slice_a.size(), slice_b.size());
+
+      return CompareSuffix<BlobEntryKey>(
+          &slice_a, &slice_b, /*only_compare_index_keys*/ false, ok);
+    }
+
     case KeyPrefix::INDEX_DATA: {
       // Provide a stable ordering for invalid data.
       if (slice_a.empty() || slice_b.empty())
         return CompareSizes(slice_a.size(), slice_b.size());
 
       return CompareSuffix<IndexDataKey>(
           &slice_a, &slice_b, only_compare_index_keys, ok);
     }
 
     case KeyPrefix::INVALID_TYPE:
@@ skipping 181 matching lines @@
   DCHECK(index_id_ != kInvalidId);
 
   if (!database_id_)
     return GLOBAL_METADATA;
   if (!object_store_id_)
     return DATABASE_METADATA;
   if (index_id_ == kObjectStoreDataIndexId)
     return OBJECT_STORE_DATA;
   if (index_id_ == kExistsEntryIndexId)
     return EXISTS_ENTRY;
+  if (index_id_ == kBlobEntryIndexId)
+    return BLOB_ENTRY;
   if (index_id_ >= kMinimumIndexId)
     return INDEX_DATA;
 
   NOTREACHED();
   return INVALID_TYPE;
 }
 
 std::string SchemaVersionKey::Encode() {
   std::string ret = KeyPrefix::EncodeEmpty();
   ret.push_back(kSchemaVersionTypeByte);
   return ret;
 }
 
 std::string MaxDatabaseIdKey::Encode() {
   std::string ret = KeyPrefix::EncodeEmpty();
   ret.push_back(kMaxDatabaseIdTypeByte);
   return ret;
 }
 
 std::string DataVersionKey::Encode() {
   std::string ret = KeyPrefix::EncodeEmpty();
   ret.push_back(kDataVersionTypeByte);
   return ret;
 }
 
+std::string BlobJournalKey::Encode()
+{
+  std::string ret = KeyPrefix::EncodeEmpty();
+  ret.push_back(kBlobJournalTypeByte);
+  return ret;
+}
+
+std::string LiveBlobJournalKey::Encode()
+{
+  std::string ret = KeyPrefix::EncodeEmpty();
+  ret.push_back(kLiveBlobJournalTypeByte);
+  return ret;
+}
+
 DatabaseFreeListKey::DatabaseFreeListKey() : database_id_(-1) {}
 
 bool DatabaseFreeListKey::Decode(StringPiece* slice,
                                  DatabaseFreeListKey* result) {
   KeyPrefix prefix;
   if (!KeyPrefix::Decode(slice, &prefix))
     return false;
   DCHECK(!prefix.database_id_);
   DCHECK(!prefix.object_store_id_);
   DCHECK(!prefix.index_id_);
@@ skipping 64 matching lines @@
   // just after origin in collation order
   return EncodeMinKeyForOrigin(origin_identifier + '\x01');
 }
 
 int DatabaseNameKey::Compare(const DatabaseNameKey& other) {
   if (int x = origin_.compare(other.origin_))
     return x;
   return database_name_.compare(other.database_name_);
 }
 
+bool DatabaseMetaDataKey::IsValidBlobKey(int64 blob_key) {
+  return blob_key >= kBlobKeyGeneratorInitialNumber;
+}
+
 std::string DatabaseMetaDataKey::Encode(int64 database_id,
                                         MetaDataType meta_data_type) {
   KeyPrefix prefix(database_id);
   std::string ret = prefix.Encode();
   ret.push_back(meta_data_type);
   return ret;
 }
 
 ObjectStoreMetaDataKey::ObjectStoreMetaDataKey()
     : object_store_id_(-1), meta_data_type_(-1) {}
@@ skipping 386 matching lines @@
   scoped_ptr<IndexedDBKey> key;
   StringPiece slice(encoded_user_key_);
   if (!DecodeIDBKey(&slice, &key)) {
     // TODO(jsbell): Return error.
   }
   return key.Pass();
 }
 
 const int64 ExistsEntryKey::kSpecialIndexNumber = kExistsEntryIndexId;
 
+bool BlobEntryKey::Decode(StringPiece* slice, BlobEntryKey* result) {
+  KeyPrefix prefix;
+  if (!KeyPrefix::Decode(slice, &prefix))
+    return false;
+  DCHECK(prefix.database_id_);
+  DCHECK(prefix.object_store_id_);
+  DCHECK_EQ(prefix.index_id_, kSpecialIndexNumber);
+
+  if (!ExtractEncodedIDBKey(slice, &result->encoded_user_key_))
+    return false;
+  result->database_id_ = prefix.database_id_;
+  result->object_store_id_ = prefix.object_store_id_;
+
+  return true;
+}
+
+bool BlobEntryKey::FromObjectStoreDataKey(StringPiece* slice,
+                                          BlobEntryKey* result) {
+  KeyPrefix prefix;
+  if (!KeyPrefix::Decode(slice, &prefix))
+    return false;
+  DCHECK(prefix.database_id_);
+  DCHECK(prefix.object_store_id_);
+  DCHECK_EQ(prefix.index_id_, ObjectStoreDataKey::kSpecialIndexNumber);
+
+  if (!ExtractEncodedIDBKey(slice, &result->encoded_user_key_))
+    return false;
+  result->database_id_ = prefix.database_id_;
+  result->object_store_id_ = prefix.object_store_id_;
+  return true;
+}
+
+std::string BlobEntryKey::EncodeToObjectStoreDataKey(StringPiece* slice) {

[jsbell, 2013/09/13 00:12:21]

What this method did wasn't obvious to me at the call site - decodes an encoded
BlobEntryKey, then re-encodes the vitals into an ObjectStoreDataKey. Maybe
"ReencodeTo..." ?

[ericu, 2013/11/20 23:05:39]

Done.

+  BlobEntryKey key;
+  if (!Decode(slice, &key))
+    return std::string();
+
+  return ObjectStoreDataKey::Encode(
+      key.database_id_, key.object_store_id_, key.encoded_user_key_);
+}
+
+std::string BlobEntryKey::EncodeMinForObjectStore(int64 database_id,
+                                                  int64 object_store_id) {
+  // Our implied encoded_user_key_ here is empty, the lowest possible key.
+  return Encode(database_id, object_store_id, std::string());
+}
+
+// This isn't technically the max for this object store--it's one higher, which
+// means that it's the first key /not/ in the range. That's more what we want,
+// but doesn't match the terminology elsewhere in this file, which is a bit
+// messy.
+std::string BlobEntryKey::EncodeMaxForObjectStore(int64 database_id,
+                                                  int64 object_store_id)
+{
+  DCHECK(KeyPrefix::ValidIds(database_id, object_store_id));
+  KeyPrefix prefix(
+      KeyPrefix::CreateWithSpecialIndex(database_id, object_store_id,
+        kSpecialIndexNumber + 1));

[jsbell, 2013/09/13 00:12:21]

Can you introduce another const, rather than kSpecialIndexNumber + 1?

[ericu, 2013/11/20 23:05:39]

I think that might be confusing.  The point is not that this is some arbitrary
constant which one could renumber at will.  In this function we have to create
the lowest key with the next-higher special index number, so that it's just
bracketing our key space.  A constant for that whole key could make things
clearer, but a constant for just the next-higher special index number seems like
it couldn't be an improvement over "kSpecialIndexNumber + 1", as it would lack
context.  If I were to add one, I'd really want to do it in this function, below
that comment, at which point I don't see the value.

[jsbell, 2013/11/21 22:54:32]

Got it, I wasn't paying attention to the usage (i.e. that this is a
EncodeMax...() func)

+  return prefix.Encode();
+}
+
+std::string BlobEntryKey::Encode() const
+{
+  DCHECK_GT(encoded_user_key_.size(), 0UL);
+  return Encode(database_id_, object_store_id_, encoded_user_key_);
+}
+
+std::string BlobEntryKey::Encode(int64 database_id,
+                                 int64 object_store_id,
+                                 const IndexedDBKey& user_key)
+{
+  std::string encoded_key;
+  EncodeIDBKey(user_key, &encoded_key);
+  return Encode(database_id, object_store_id, encoded_key);
+}
+
+std::string BlobEntryKey::Encode(
+    int64 database_id, int64 object_store_id,
+    const std::string& encoded_user_key) {
+  DCHECK(KeyPrefix::ValidIds(database_id, object_store_id));
+  KeyPrefix prefix(
+      KeyPrefix::CreateWithSpecialIndex(database_id, object_store_id,
+                                        kSpecialIndexNumber));
+  std::string ret = prefix.Encode();
+  ret.insert(ret.end(), encoded_user_key.begin(), encoded_user_key.end());
+  return ret;
+}
+
+int BlobEntryKey::Compare(const BlobEntryKey& other, bool* ok)
+{
+  DCHECK_GT(encoded_user_key_.size(), 0UL);
+  DCHECK_GT(other.encoded_user_key_.size(), 0UL);
+  return CompareEncodedIDBKeys(encoded_user_key_, other.encoded_user_key_, ok);
+}
+
+const int64 BlobEntryKey::kSpecialIndexNumber = kBlobEntryIndexId;
+
 IndexDataKey::IndexDataKey()
     : database_id_(-1),
       object_store_id_(-1),
       index_id_(-1),
       sequence_number_(-1) {}
 
 IndexDataKey::~IndexDataKey() {}
 
 bool IndexDataKey::Decode(StringPiece* slice, IndexDataKey* result) {
   KeyPrefix prefix;
@@ skipping 113 matching lines @@
 scoped_ptr<IndexedDBKey> IndexDataKey::primary_key() const {
   scoped_ptr<IndexedDBKey> key;
   StringPiece slice(encoded_primary_key_);
   if (!DecodeIDBKey(&slice, &key)) {
     // TODO(jsbell): Return error.
   }
   return key.Pass();
 }
 
 }  // namespace content