Blob support for IDB [Chromium]

content/browser/indexed_db/indexed_db_leveldb_coding.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/indexed_db/indexed_db_leveldb_coding.h"
 
 #include <iterator>
 #include <limits>
 
 #include "base/logging.h"
@@ skipping 26 matching lines @@
 // Strings (origins, names, etc) are encoded as UTF-16BE.
 //
 //
 // Global metadata
 // ---------------
 // The prefix is <0, 0, 0>, followed by a metadata type byte:
 //
 // <0, 0, 0, 0> => backing store schema version [SchemaVersionKey]
 // <0, 0, 0, 1> => maximum allocated database [MaxDatabaseIdKey]
 // <0, 0, 0, 2> => SerializedScriptValue version [DataVersionKey]
+// <0, 0, 0, 3>
+//   => Blob journal
+//     The format of the journal is: {database_id, blobKey}*.
+//     If the blobKey is AllBlobsKey, the whole database should be deleted.
+//     [BlobJournalKey]
+// <0, 0, 0, 4> => Live blob journal; same format. [LiveBlobJournalKey]
 // <0, 0, 0, 100, database id>
 //   => Existence implies the database id is in the free list
 //      [DatabaseFreeListKey]
 // <0, 0, 0, 201, origin, database name> => Database id [DatabaseNameKey]
 //
 //
 // Database metadata: [DatabaseMetaDataKey]
 // ----------------------------------------
 // The prefix is <database id, 0, 0> followed by a metadata type byte:
 //
 // <database id, 0, 0, 0> => origin name
 // <database id, 0, 0, 1> => database name
 // <database id, 0, 0, 2> => IDB string version data (obsolete)
 // <database id, 0, 0, 3> => maximum allocated object store id
 // <database id, 0, 0, 4> => IDB integer version (var int)
+// <database id, 0, 0, 5> => blob key generator current number
 //
 //
 // Object store metadata: [ObjectStoreMetaDataKey]
 // -----------------------------------------------
 // The prefix is <database id, 0, 0>, followed by a type byte (50), then the
 // object store id (var int), then a metadata type byte.
 //
 // <database id, 0, 0, 50, object store id, 0> => object store name
 // <database id, 0, 0, 50, object store id, 1> => key path
 // <database id, 0, 0, 50, object store id, 2> => auto increment flag
@@ skipping 54 matching lines @@
 //   => "version", serialized script value
 //
 //
 // "Exists" entry: [ExistsEntryKey]
 // --------------------------------
 // The prefix is followed by a type byte and the encoded IDB primary key.
 //
 // <database id, object store id, 2, user key> => "version"
 //
 //
+// Blob entry table: [BlobEntryKey]
+// --------------------------------
+//
+// The prefix is followed by a type byte and the encoded IDB primary key.
+//
+// <database id, object store id, 3, user key> => array of IndexedDBBlobInfo
+//
+//
 // Index data
 // ----------
 // The prefix is followed by a type byte, the encoded IDB index key, a
 // "sequence" number (obsolete; var int), and the encoded IDB primary key.
 //
 // <database id, object store id, index id, index key, sequence number,
 //   primary key> => "version", primary key [IndexDataKey]
 //
 // The sequence number is obsolete; it was used to allow two entries with the
 // same user (index) key in non-unique indexes prior to the inclusion of the
 // primary key in the data.
 
-
 using base::StringPiece;
 using blink::WebIDBKeyType;
 using blink::WebIDBKeyTypeArray;
 using blink::WebIDBKeyTypeBinary;
 using blink::WebIDBKeyTypeDate;
 using blink::WebIDBKeyTypeInvalid;
 using blink::WebIDBKeyTypeMin;
 using blink::WebIDBKeyTypeNull;
 using blink::WebIDBKeyTypeNumber;
 using blink::WebIDBKeyTypeString;
 using blink::WebIDBKeyPathType;
 using blink::WebIDBKeyPathTypeArray;
 using blink::WebIDBKeyPathTypeNull;
 using blink::WebIDBKeyPathTypeString;
 
 namespace content {
 
 // As most of the IndexedDBKeys and encoded values are short, we
-// initialize some Vectors with a default inline buffer size to reduce
-// the memory re-allocations when the Vectors are appended.
+// initialize some std::vectors with a default inline buffer size to reduce
+// the memory re-allocations when the std::vectors are appended.
 static const size_t kDefaultInlineBufferSize = 32;
 
 static const unsigned char kIndexedDBKeyNullTypeByte = 0;
 static const unsigned char kIndexedDBKeyStringTypeByte = 1;
 static const unsigned char kIndexedDBKeyDateTypeByte = 2;
 static const unsigned char kIndexedDBKeyNumberTypeByte = 3;
 static const unsigned char kIndexedDBKeyArrayTypeByte = 4;
 static const unsigned char kIndexedDBKeyMinKeyTypeByte = 5;
 static const unsigned char kIndexedDBKeyBinaryTypeByte = 6;
 
 static const unsigned char kIndexedDBKeyPathTypeCodedByte1 = 0;
 static const unsigned char kIndexedDBKeyPathTypeCodedByte2 = 0;
 
 static const unsigned char kObjectStoreDataIndexId = 1;
 static const unsigned char kExistsEntryIndexId = 2;
+static const unsigned char kBlobEntryIndexId = 3;
 
 static const unsigned char kSchemaVersionTypeByte = 0;
 static const unsigned char kMaxDatabaseIdTypeByte = 1;
 static const unsigned char kDataVersionTypeByte = 2;
+static const unsigned char kBlobJournalTypeByte = 3;
+static const unsigned char kLiveBlobJournalTypeByte = 4;
 static const unsigned char kMaxSimpleGlobalMetaDataTypeByte =
-    3;  // Insert before this and increment.
+    5;  // Insert before this and increment.
 static const unsigned char kDatabaseFreeListTypeByte = 100;
 static const unsigned char kDatabaseNameTypeByte = 201;
 
 static const unsigned char kObjectStoreMetaDataTypeByte = 50;
 static const unsigned char kIndexMetaDataTypeByte = 100;
 static const unsigned char kObjectStoreFreeListTypeByte = 150;
 static const unsigned char kIndexFreeListTypeByte = 151;
 static const unsigned char kObjectStoreNamesTypeByte = 200;
 static const unsigned char kIndexNamesKeyTypeByte = 201;
 
@@ skipping 682 matching lines @@
 
 template <>
 int CompareSuffix<ObjectStoreDataKey>(StringPiece* slice_a,
                                       StringPiece* slice_b,
                                       bool only_compare_index_keys,
                                       bool* ok) {
   return CompareEncodedIDBKeys(slice_a, slice_b, ok);
 }
 
 template <>
+int CompareSuffix<BlobEntryKey>(StringPiece* slice_a,
+                                StringPiece* slice_b,
+                                bool only_compare_index_keys,
+                                bool* ok) {
+  return CompareEncodedIDBKeys(slice_a, slice_b, ok);
+}
+
+template <>
 int CompareSuffix<IndexDataKey>(StringPiece* slice_a,
                                 StringPiece* slice_b,
                                 bool only_compare_index_keys,
                                 bool* ok) {
   // index key
   int result = CompareEncodedIDBKeys(slice_a, slice_b, ok);
   if (!*ok || result)
     return result;
   if (only_compare_index_keys)
     return 0;
@@ skipping 130 matching lines @@
 
     case KeyPrefix::EXISTS_ENTRY: {
       // Provide a stable ordering for invalid data.
       if (slice_a.empty() || slice_b.empty())
         return CompareSizes(slice_a.size(), slice_b.size());
 
       return CompareSuffix<ExistsEntryKey>(
           &slice_a, &slice_b, /*only_compare_index_keys*/ false, ok);
     }
 
+    case KeyPrefix::BLOB_ENTRY: {
+      // Provide a stable ordering for invalid data.
+      if (slice_a.empty() || slice_b.empty())
+        return CompareSizes(slice_a.size(), slice_b.size());
+
+      return CompareSuffix<BlobEntryKey>(
+          &slice_a, &slice_b, /*only_compare_index_keys*/ false, ok);
+    }
+
     case KeyPrefix::INDEX_DATA: {
       // Provide a stable ordering for invalid data.
       if (slice_a.empty() || slice_b.empty())
         return CompareSizes(slice_a.size(), slice_b.size());
 
       return CompareSuffix<IndexDataKey>(
           &slice_a, &slice_b, only_compare_index_keys, ok);
     }
 
     case KeyPrefix::INVALID_TYPE:
@@ skipping 181 matching lines @@
   DCHECK(index_id_ != kInvalidId);
 
   if (!database_id_)
     return GLOBAL_METADATA;
   if (!object_store_id_)
     return DATABASE_METADATA;
   if (index_id_ == kObjectStoreDataIndexId)
     return OBJECT_STORE_DATA;
   if (index_id_ == kExistsEntryIndexId)
     return EXISTS_ENTRY;
+  if (index_id_ == kBlobEntryIndexId)
+    return BLOB_ENTRY;
   if (index_id_ >= kMinimumIndexId)
     return INDEX_DATA;
 
   NOTREACHED();
   return INVALID_TYPE;
 }
 
 std::string SchemaVersionKey::Encode() {
   std::string ret = KeyPrefix::EncodeEmpty();
   ret.push_back(kSchemaVersionTypeByte);
   return ret;
 }
 
 std::string MaxDatabaseIdKey::Encode() {
   std::string ret = KeyPrefix::EncodeEmpty();
   ret.push_back(kMaxDatabaseIdTypeByte);
   return ret;
 }
 
 std::string DataVersionKey::Encode() {
   std::string ret = KeyPrefix::EncodeEmpty();
   ret.push_back(kDataVersionTypeByte);
   return ret;
 }
 
+std::string BlobJournalKey::Encode() {
+  std::string ret = KeyPrefix::EncodeEmpty();
+  ret.push_back(kBlobJournalTypeByte);
+  return ret;
+}
+
+std::string LiveBlobJournalKey::Encode() {
+  std::string ret = KeyPrefix::EncodeEmpty();
+  ret.push_back(kLiveBlobJournalTypeByte);
+  return ret;
+}
+
 DatabaseFreeListKey::DatabaseFreeListKey() : database_id_(-1) {}
 
 bool DatabaseFreeListKey::Decode(StringPiece* slice,
                                  DatabaseFreeListKey* result) {
   KeyPrefix prefix;
   if (!KeyPrefix::Decode(slice, &prefix))
     return false;
   DCHECK(!prefix.database_id_);
   DCHECK(!prefix.object_store_id_);
   DCHECK(!prefix.index_id_);
@@ skipping 64 matching lines @@
   // just after origin in collation order
   return EncodeMinKeyForOrigin(origin_identifier + '\x01');
 }
 
 int DatabaseNameKey::Compare(const DatabaseNameKey& other) {
   if (int x = origin_.compare(other.origin_))
     return x;
   return database_name_.compare(other.database_name_);
 }
 
+bool DatabaseMetaDataKey::IsValidBlobKey(int64 blob_key) {
+  return blob_key >= kBlobKeyGeneratorInitialNumber;
+}
+
 std::string DatabaseMetaDataKey::Encode(int64 database_id,
                                         MetaDataType meta_data_type) {
   KeyPrefix prefix(database_id);
   std::string ret = prefix.Encode();
   ret.push_back(meta_data_type);
   return ret;
 }
 
 ObjectStoreMetaDataKey::ObjectStoreMetaDataKey()
     : object_store_id_(-1), meta_data_type_(-1) {}
@@ skipping 386 matching lines @@
   scoped_ptr<IndexedDBKey> key;
   StringPiece slice(encoded_user_key_);
   if (!DecodeIDBKey(&slice, &key)) {
     // TODO(jsbell): Return error.
   }
   return key.Pass();
 }
 
 const int64 ExistsEntryKey::kSpecialIndexNumber = kExistsEntryIndexId;
 
+bool BlobEntryKey::Decode(StringPiece* slice, BlobEntryKey* result) {
+  KeyPrefix prefix;
+  if (!KeyPrefix::Decode(slice, &prefix))
+    return false;
+  DCHECK(prefix.database_id_);
+  DCHECK(prefix.object_store_id_);
+  DCHECK_EQ(prefix.index_id_, kSpecialIndexNumber);
+
+  if (!ExtractEncodedIDBKey(slice, &result->encoded_user_key_))
+    return false;
+  result->database_id_ = prefix.database_id_;
+  result->object_store_id_ = prefix.object_store_id_;
+
+  return true;
+}
+
+bool BlobEntryKey::FromObjectStoreDataKey(StringPiece* slice,
+                                          BlobEntryKey* result) {
+  KeyPrefix prefix;
+  if (!KeyPrefix::Decode(slice, &prefix))
+    return false;
+  DCHECK(prefix.database_id_);
+  DCHECK(prefix.object_store_id_);
+  DCHECK_EQ(prefix.index_id_, ObjectStoreDataKey::kSpecialIndexNumber);
+
+  if (!ExtractEncodedIDBKey(slice, &result->encoded_user_key_))
+    return false;
+  result->database_id_ = prefix.database_id_;
+  result->object_store_id_ = prefix.object_store_id_;
+  return true;
+}
+
+std::string BlobEntryKey::ReencodeToObjectStoreDataKey(StringPiece* slice) {
+  BlobEntryKey key;
+  if (!Decode(slice, &key))
+    return std::string();
+
+  return ObjectStoreDataKey::Encode(
+      key.database_id_, key.object_store_id_, key.encoded_user_key_);
+}
+
+std::string BlobEntryKey::EncodeMinForObjectStore(int64 database_id,
+                                                  int64 object_store_id) {
+  // Our implied encoded_user_key_ here is empty, the lowest possible key.
+  return Encode(database_id, object_store_id, std::string());
+}
+
+// This isn't technically the max for this object store--it's one higher, which

[jsbell, 2013/12/20 00:44:20]

I think there are a few other cases like this (where "max" is
open/exclusive/outside the range), but agreed that the file doesn't make this
clear. A renaming patch would be great.

[ericu, 2014/02/20 00:50:29]

I see it's called the "stop key" elsewhere, so I've switched to that.

+// means that it's the first key /not/ in the range. That's more what we want,
+// but doesn't match the terminology elsewhere in this file, which is a bit
+// messy.
+std::string BlobEntryKey::EncodeMaxForObjectStore(int64 database_id,
+                                                  int64 object_store_id) {
+  DCHECK(KeyPrefix::ValidIds(database_id, object_store_id));
+  KeyPrefix prefix(KeyPrefix::CreateWithSpecialIndex(
+      database_id, object_store_id, kSpecialIndexNumber + 1));
+  return prefix.Encode();
+}
+
+std::string BlobEntryKey::Encode() const {
+  DCHECK_GT(encoded_user_key_.size(), 0UL);
+  return Encode(database_id_, object_store_id_, encoded_user_key_);
+}
+
+std::string BlobEntryKey::Encode(int64 database_id,
+                                 int64 object_store_id,
+                                 const IndexedDBKey& user_key) {
+  std::string encoded_key;
+  EncodeIDBKey(user_key, &encoded_key);
+  return Encode(database_id, object_store_id, encoded_key);
+}
+
+std::string BlobEntryKey::Encode(int64 database_id,
+                                 int64 object_store_id,
+                                 const std::string& encoded_user_key) {
+  DCHECK(KeyPrefix::ValidIds(database_id, object_store_id));
+  KeyPrefix prefix(KeyPrefix::CreateWithSpecialIndex(
+      database_id, object_store_id, kSpecialIndexNumber));
+  std::string ret = prefix.Encode();
+  ret.insert(ret.end(), encoded_user_key.begin(), encoded_user_key.end());
+  return ret;
+}
+
+int BlobEntryKey::Compare(const BlobEntryKey& other, bool* ok) {

[jsbell, 2013/12/20 00:44:20]

This is probably unused (outside of tests?) I think we recently deleted a bunch
of these c/o dead code detection tool, and only kept FooKey::Compare() where
CompareSuffix<FooKey>() was not used.

[ericu, 2014/02/20 00:50:29]

Done.

+  DCHECK_GT(encoded_user_key_.size(), 0UL);
+  DCHECK_GT(other.encoded_user_key_.size(), 0UL);
+  return CompareEncodedIDBKeys(encoded_user_key_, other.encoded_user_key_, ok);
+}
+
+const int64 BlobEntryKey::kSpecialIndexNumber = kBlobEntryIndexId;
+
 IndexDataKey::IndexDataKey()
     : database_id_(-1),
       object_store_id_(-1),
       index_id_(-1),
       sequence_number_(-1) {}
 
 IndexDataKey::~IndexDataKey() {}
 
 bool IndexDataKey::Decode(StringPiece* slice, IndexDataKey* result) {
   KeyPrefix prefix;
@@ skipping 130 matching lines @@
 scoped_ptr<IndexedDBKey> IndexDataKey::primary_key() const {
   scoped_ptr<IndexedDBKey> key;
   StringPiece slice(encoded_primary_key_);
   if (!DecodeIDBKey(&slice, &key)) {
     // TODO(jsbell): Return error.
   }
   return key.Pass();
 }
 
 }  // namespace content