Wrap large IndexedDB values into Blobs before writing to LevelDB.

third_party/WebKit/Source/modules/indexeddb/IDBValueWrapping.h

Index: third_party/WebKit/Source/modules/indexeddb/IDBValueWrapping.h
diff --git a/third_party/WebKit/Source/modules/indexeddb/IDBValueWrapping.h b/third_party/WebKit/Source/modules/indexeddb/IDBValueWrapping.h
new file mode 100644
index 0000000000000000000000000000000000000000..d5cdce8d663dcd6603bec547cc08ce7e83787b9c
--- /dev/null
+++ b/third_party/WebKit/Source/modules/indexeddb/IDBValueWrapping.h
@@ -0,0 +1,188 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef IDBValueWrapping_h
+#define IDBValueWrapping_h
+
+#include "bindings/core/v8/ExceptionState.h"
+#include "bindings/core/v8/serialization/SerializedScriptValue.h"
+#include "platform/SharedBuffer.h"
+#include "platform/wtf/Allocator.h"
+#include "platform/wtf/RefPtr.h"
+#include "platform/wtf/Vector.h"
+#include "public/platform/WebBlobInfo.h"
+#include "v8/include/v8.h"
+
+namespace blink {
+
+class Blob;
+class BlobDataHandle;
+class ExceptionState;
+class IDBValue;
+class ScriptState;
+class ScriptValue;
+class SerializedScriptValue;
+class SharedBuffer;
+
+// Logic for serializing V8 values for storage in IndexedDB.
+//
+// V8 values are stored on disk using the format implemented in
+// SerializedScriptValue (SSV). For "normal" (not too large) V8 values, the SSV
+// output is stored directly in IndexedDB's backing store. "Large" V8 values are
+// wrapped in Blobs, in order to avoid operating the backing store in a
+// sub-optimal region. In this case, the SSV output is stored using the Blob
+// storage system, and IndexedDB's backing store only holds references to the
+// Blobs.
+//
+// In summary:
+// "normal" v8::Value -> SSV -> IDBValue (stores SSV output) -> LevelDB
+// "large" v8::Value -> SSV -> IDBValue (stores SSV output) ->
+//     Blob (stores SSV output) + IDBValue (stores Blob reference) -> LevelDB
+class IDBValueWrapper {
+  STACK_ALLOCATED();
+
+ public:
+  // Wrapper for an IndexedDB value.
+  //
+  // The serialization process can throw an exception. The caller is responsible
+  // for checking exception_state.
+  IDBValueWrapper(
+      v8::Isolate*,
+      v8::Local<v8::Value>,
+      SerializedScriptValue::SerializeOptions::WasmSerializationPolicy,
+      ExceptionState&);
+
+  // Creates a clone of the serialized value.
+  //
+  // This method is used to fulfill the IndexedDB specification requirement that
+  // a value's key and index keys are extracted from a structured clone of the
+  // value, which avoids the issue of side-effects in custom getters.
+  //
+  // This method cannot be called after WrapIfBiggerThan().
+  void Clone(ScriptState*, ScriptValue* clone);
+
+  // Wraps the serialized value into a Blob if it exceeds the given size.
+  //
+  // In production, the threshold is currently always kWrapThreshold.
+  //
+  // This method must be called before ExtractWireBytes() and cannot be called
+  // after ExtractWireBytes().
+  bool WrapIfBiggerThan(unsigned max_bytes);
+
+  void ExtractBlobDataHandles(
+      Vector<RefPtr<BlobDataHandle>>* blob_data_handles);
+
+  RefPtr<SharedBuffer> ExtractWireBytes();
+
+  inline Vector<WebBlobInfo>& WrappedBlobInfo() {
+#if DCHECK_IS_ON()
+    DCHECK(!had_exception_)
+        << "WrapBlobInfo() called on wrapper with serialization exception";
+#endif  // DCHECK_IS_ON()
+    return blob_info_;
+  }
+
+  // Default threshold for WrapIfBiggerThan().
+  //
+  // This should be tuned to achieve a compromise between short-term IndexedDB
+  // throughput and long-term I/O load and memory usage. LevelDB, the underlying
+  // storage for IndexedDB, was not designed with large values in mind. At the
+  // very least, large values will slow down compaction, causing occasional I/O
+  // spikes.
+  static constexpr unsigned kWrapThreshold = 64 * 1024;
+
+  // MIME type used for Blobs that wrap IDBValues.
+  static constexpr const char* kWrapMimeType =
+      "application/x-blink-idb-value-wrapper";

[jbroman, 2017/05/23 20:13:11]

IIUC, IANA no longer recommends deploying unregistered MIME types with the x-
prefix.
https://tools.ietf.org/html/rfc6838

Dunno if it's worth registering this, using x. instead of x-, or just deciding
this is never going to be visible and thus we don't care. (Or we could use
application/octet-stream, I suppose.)

[pwnall, 2017/05/25 13:27:12]

I submitted an application for "application/vnd.blink-idb-value-wrapper".

Thank you very much for pointing this out! I had no idea that things have
changed :)

While I don't expect this to be user-visible, I see no harm in using a
registered MIME type to tag our Blobs. This might help developers if the Blobs
ever end up in a developer-visible UI, whether on purpose or by accident.

+
+ private:
+  // Used to serialize the wrapped value.
+  static void WriteVarint(unsigned value, Vector<char>& output);
+
+  RefPtr<SerializedScriptValue> serialized_value_;
+  RefPtr<BlobDataHandle> wrapper_handle_;
+  Vector<WebBlobInfo> blob_info_;
+  Vector<BlobDataHandle> blob_handles_;
+  Vector<char> wire_bytes_;
+#if DCHECK_IS_ON()
+  bool had_exception_ = false;
+  bool wrap_called_ = false;
+#endif  // DCHECK_IS_ON()
+};
+
+// State and logic for unwrapping large IndexedDB values from Blobs.
+//
+// See IDBValueWrapper for an explanation of the wrapping concept.
+//
+// Once created, an IDBValueUnwrapper instance can be used to unwrap multiple
+// Blobs. For each Blob to be unwrapped, the caller should first call Parse().
+// If the method succeeds, the IDBValueUnwrapper will store the parse state,
+// which can be obtained using WrapperBlobSize() and WrapperBlobHandle().
+class IDBValueUnwrapper {
+  STACK_ALLOCATED();
+
+ public:
+  IDBValueUnwrapper();
+
+  // True if the IDBValue's data was wrapped in a Blob.
+  static bool IsWrapped(IDBValue*);
+
+  // True if at least one of the IDBValues' data was wrapped in a Blob.
+  static bool IsWrapped(const Vector<RefPtr<IDBValue>>&);
+
+  // Pieces together an unwrapped IDBValue from a wrapped value and Blob data.
+  static RefPtr<IDBValue> CreateUnwrapped(
+      IDBValue* wrapped_value,
+      RefPtr<SharedBuffer>&& wrapper_blob_content);
+
+  // Parses the wrapper Blob information from a wrapped IDBValue.
+  //
+  // Returns true for success, and false for failure. Failure can mean that the
+  // given value was not a wrapped IDBValue, or that the value bytes were
+  // corrupted.
+  bool Parse(IDBValue*);
+
+  // Returns the size of the Blob obtained by the last Unwrap() call.
+  //
+  // Should only be called after a successful result from Unwrap().
+  inline unsigned WrapperBlobSize() const {
+    DCHECK(end_);
+    return blob_size_;
+  }
+
+  // Returns a handle to the Blob obtained by the last Unwrap() call.
+  //
+  // Should only be called exactly once after a successful result from Unwrap().
+  RefPtr<BlobDataHandle> WrapperBlobHandle();
+
+ private:
+  // Used to deserialize the wrapped value.
+  bool ReadVarint(unsigned& value);
+
+  // Resets the parsing state.
+  inline bool Reset() {
+#if DCHECK_IS_ON()
+    blob_handle_.Clear();
+    current_ = nullptr;
+    end_ = nullptr;
+#endif  // DCHECK_IS_ON()
+    return false;
+  }
+
+  // Deserialization cursor in the SharedBuffer of the IDBValue being unwrapped.
+  const uint8_t* current_;
+
+  // Smallest invalid position_ value.
+  const uint8_t* end_;
+
+  // The size of the Blob holding the data for the last unwrapped IDBValue.
+  unsigned blob_size_;
+
+  // Handle to the Blob holding the data for the last unwrapped IDBValue.
+  RefPtr<BlobDataHandle> blob_handle_;
+};
+
+}  // namespace blink
+
+#endif  // IDBValueWrapping_h