Wrap large IndexedDB values into Blobs before writing to LevelDB.

third_party/WebKit/Source/modules/indexeddb/IDBValueWrapping.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef IDBValueWrapping_h
+#define IDBValueWrapping_h
+
+#include "bindings/core/v8/ExceptionState.h"
+#include "platform/SharedBuffer.h"
+#include "platform/wtf/RefPtr.h"
+#include "platform/wtf/Vector.h"
+#include "public/platform/WebBlobInfo.h"
+#include "v8/include/v8.h"
+
+namespace WTF {
+class String;
+}
+
+namespace blink {
+
+class Blob;
+class BlobDataHandle;
+class ExceptionState;
+class IDBValue;
+class ScriptState;
+class ScriptValue;
+class SerializedScriptValue;
+class SharedBuffer;
+
+// Logic for serializing V8 values for storage in IndexedDB.
+//
+// "Normal" V8 values are stored on disk using the format implemented in

[jsbell, 2017/05/15 23:37:39]

nit: this first sentence is true whether we wrap or not. Rephrase?

[pwnall, 2017/05/19 18:27:35]

Done.

+// SerializedScriptValue. In order to avoid operating IndexedDB's backing store
+// in a sub-optimal region, "large" V8 values are wrapped in Blobs. In this
+// case, the SSV output is stored using the Blob storage system, and IndexedDB's
+// backing store only holds references to the Blobs.
+//
+// In summary:
+// "normal" v8::Value -> SSV -> IDBValue (stores SSV output) -> LevelDB
+// "large" v8::Value -> SSV -> IDBValue (stores SSV output) ->
+//     Blob (stores SSV output) + IDBValue (stores Blob reference) -> LevelDB
+class IDBValueWrapper {
+ public:
+  // Wrapper for an IndexedDB value.
+  //
+  // The serialization process can throw an exception. The caller is responsible
+  // for checking exception_state.
+  IDBValueWrapper(v8::Isolate*,
+                  v8::Local<v8::Value>,
+                  bool write_wasm_to_stream,
+                  ExceptionState&);
+
+  // Creates a clone of the serialized value.
+  //
+  // This method is necessary because the IndexedDB specification demands that a
+  // value's key and index keys are extracted from a structured clone of the
+  // value.

[jsbell, 2017/05/15 23:37:39]

Add "... to avoid side effects."

(We didn't spec it for fun!)

[pwnall, 2017/05/19 18:27:35]

Done.

Yikes, I didn't mean to criticize the spec, I only intended to justify myself
(w/o knowing the spec, cloning functionality seems unnecessary in serialization
logic). FWIW, I think that the structured clone solution is fairly elegant for
spec-land, especially as (IIUC) there's room for an optimized implementation to
skip creating the clone.

+  //
+  // This method cannot be called after WrapIfBiggerThan().
+  void Clone(ScriptState*, ScriptValue* clone);
+
+  // Wraps the serialized value into a Blob if it exceeds the given size.
+  //
+  // In production, the threshold is currently always kWrapThreshold.
+  //
+  // This method must be called before ExtractWireBytes() and cannot be called
+  // after ExtractWireBytes().
+  bool WrapIfBiggerThan(unsigned max_bytes);
+
+  void ExtractBlobDataHandles(
+      Vector<RefPtr<BlobDataHandle>>* blob_data_handles);
+
+  RefPtr<SharedBuffer> ExtractWireBytes();
+
+  inline Vector<WebBlobInfo>& WrappedBlobInfo() {
+#if DCHECK_IS_ON()
+    DCHECK(!had_exception_)
+        << "WrapBlobInfo() called on wrapper with serialization exception";
+#endif  // DCHECK_IS_ON()
+    return blob_info_;
+  }
+
+  // Default threshold for WrapIfBiggerThan().
+  //
+  // This should be tuned to achieve a compromise between short-term IndexedDB
+  // throughput and long-term I/O load and memory usage. LevelDB, the underlying
+  // storage for IndexedDB, was not designed with large values in mind. At the
+  // very least, large values will slow down compaction, causing occasional I/O
+  // spikes.
+  static constexpr unsigned kWrapThreshold = 64 * 1024;
+
+  // MIME type used for Blobs that wrap IDBValues.
+  static constexpr const char* kWrapMimeType =
+      "application/x-blink-idb-value-wrapper";
+
+ private:
+  // Used to serialize the wrapped value.
+  static void WriteVarint(unsigned value, Vector<char>& output);
+  static void WriteAsciiString(const String& value, Vector<char>& output);
+
+  RefPtr<SerializedScriptValue> serialized_value_;
+  RefPtr<BlobDataHandle> wrapper_handle_;
+  Vector<WebBlobInfo> blob_info_;
+  Vector<BlobDataHandle> blob_handles_;
+  Vector<char> wire_bytes_;
+#if DCHECK_IS_ON()
+  bool had_exception_ = false;
+  bool wrap_called_ = false;
+#endif  // DCHECK_IS_ON()
+};
+
+// State and logic for unwrapping large IndexedDB values from Blobs.
+//
+// See IDBValueWrapper for an explanation of the wrapping concept.
+//
+// Once created, an IDBValueUnwrapper instance can be used to unwrap multiple
+// Blobs. For each Blob to be unwrapped, the caller should first call Parse().
+// If the method succeeds, the IDBValueUnwrapper will store the parse state,
+// which can be obtained using WrapperBlobSize() and WrapperBlobHandle().
+class IDBValueUnwrapper {
+ public:
+  IDBValueUnwrapper();
+
+  // Checks whether a IDBValue's data was wrapped in a Blob.
+  static bool IsWrapped(IDBValue*);
+
+  // Pieces together an unwrapped IDBValue from a wrapped value and Blob data.
+  static RefPtr<IDBValue> CreateUnwrapped(
+      IDBValue* wrapped_value,
+      RefPtr<SharedBuffer>&& wrapper_blob_content);
+
+  // Parses the wrapper Blob information from a wrapped IDBValue.
+  //
+  // Returns true for success, and false for failure. Failure can mean that the
+  // given value was not a wrapped IDBValue, or that the value bytes were
+  // corrupted.
+  bool Parse(IDBValue*);
+
+  // Returns the size of the Blob obtained by the last Unwrap() call.
+  //
+  // Should only be called after a successful result from Unwrap().
+  inline unsigned WrapperBlobSize() const {
+    DCHECK(end_);
+    return blob_size_;
+  }
+
+  // Returns a handle to the Blob obtained by the last Unwrap() call.
+  //
+  // Should only be called exactly once after a successful result from Unwrap().
+  RefPtr<BlobDataHandle> WrapperBlobHandle();
+
+ private:
+  // Used to deserialize the wrapped value.
+  bool ReadVarint(unsigned& value);
+  bool ReadAsciiString(String& value);
+
+  // Resets the parsing state.
+  inline bool Reset() {
+#if DCHECK_IS_ON()
+    blob_handle_.Clear();
+    current_ = nullptr;
+    end_ = nullptr;
+#endif  // DCHECK_IS_ON()
+    return false;
+  }
+
+  // Deserialization cursor in the SharedBuffer of the IDBValue being unwrapped.
+  const uint8_t* current_;
+
+  // Smallest invalid position_ value.
+  const uint8_t* end_;
+
+  // The size of the Blob holding the data for the last unwrapped IDBValue.
+  unsigned blob_size_;
+
+  // Handle to the Blob holding the data for the last unwrapped IDBValue.
+  RefPtr<BlobDataHandle> blob_handle_;
+};
+
+}  // namespace blink
+
+#endif  // IDBValueWrapping_h