Wrap large IndexedDB values into Blobs before writing to LevelDB.

third_party/WebKit/Source/modules/indexeddb/IDBValueWrapping.cpp

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "modules/indexeddb/IDBValueWrapping.h"
+
+#include <utility>
+
+#include "bindings/core/v8/ScriptValue.h"
+#include "bindings/core/v8/serialization/SerializationTag.h"
+#include "bindings/core/v8/serialization/SerializedScriptValue.h"
+#include "bindings/modules/v8/V8BindingForModules.h"
+#include "core/fileapi/Blob.h"
+#include "modules/indexeddb/IDBRequest.h"
+#include "modules/indexeddb/IDBValue.h"
+#include "platform/blob/BlobData.h"
+#include "platform/wtf/text/WTFString.h"
+
+namespace blink {
+
+namespace {
+
+// V8 values are stored on disk by IndexedDB using the format implemented in
+// SerializedScriptValue (SSV). The wrapping detection logic in
+// IDBValueUnwrapper::IsWrapped() must be able to distinguish between SSV byte
+// sequences produced and byte sequences expressing the fact that an IDBValue
+// has been wrapped and requires post-processing.
+//
+// The detection logic takes advantage of the highly regular structure around
+// SerializedScriptValue. A version 17 byte sequence always starts with the
+// following four bytes:
+//
+// 1) 0xFF - version tag
+// 2) 0x11 - Blink wrapper version, 17
+// 3) 0xFF - version tag
+// 4) 0x0D - V8 serialization version, currently 13, doesn't matter
+//
+// It follows that SSV will never produce byte sequences starting with 0xFF,

[jsbell, 2017/05/15 23:37:38]

Can you run this past jbroman@ as a sanity check?

[pwnall, 2017/05/19 18:27:35]

Will do.

+// 0x11, and any value except for 0xFF. If the SSV format changes, the version
+// will have to be bumped.
+
+// The SSV format version whose encoding hole is (ab)used for wrapping.
+const static uint8_t kRequiresProcessingSSVPseudoVersion = 17;
+
+// Identifies IndexedDB values that were wrapped in Blobs. The wrapper has the
+// following format:
+//
+// 1) varint - Blob size
+// 2) varint length-prefixed ASCII string - Blob UUID

[jsbell, 2017/05/15 23:37:39]

This is where I unhelpfully wonder if just storing the 16 byte UUID data is
better than storing 37 bytes (1 byte length = 0x24 + 36 ASCII char
serialization)

[pwnall, 2017/05/19 18:27:35]

FWIW, I think this (serialization format) is the most important area of the CL.

After realizing that Blobs in IndexedDB don't come back with the same UUID, I
dropped the UUID from the serialization format. Instead, I'm adding the index in
the list of Blobs. This is redundant (as was the UUID), because the Blob that
holds theSSV is always the last Blob in the list, and the current unwrapping
code takes advantage of this observation.

Justification for the pieces I have:
1) Blob size -- we may want to use different unwrapping methods / schedule
things differently based on the Blob size
2) Blob index -- I imagine it might come in handy in the future. For example, we
could tack on more Blobs (and more data to the wrapper Blob) without changing
the kBlobWrappedValue tag.

[jsbell, 2017/05/22 21:54:19]

sgtm...

On 2017/05/19 18:27:35, pwnall wrote:
I didn't actually see this documented anywhere?

+const static uint8_t kBlobWrappedValue = 1;
+
+}  // namespace
+
+IDBValueWrapper::IDBValueWrapper(v8::Isolate* isolate,
+                                 v8::Local<v8::Value> value,
+                                 bool write_wasm_to_stream,
+                                 ExceptionState& exception_state) {
+  SerializedScriptValue::SerializeOptions options;
+  options.blob_info = &blob_info_;
+  options.for_storage = SerializedScriptValue::kForStorage;
+  options.write_wasm_to_stream = write_wasm_to_stream;
+
+  serialized_value_ = SerializedScriptValue::Serialize(isolate, value, options,
+                                                       exception_state);
+
+#if DCHECK_IS_ON()
+  if (exception_state.HadException())
+    had_exception_ = true;
+#endif  // DCHECK_IS_ON()
+}
+
+void IDBValueWrapper::Clone(ScriptState* script_state, ScriptValue* clone) {
+#if DCHECK_IS_ON()
+  DCHECK(!had_exception_) << __FUNCTION__
+                          << " called on wrapper with serialization exception";
+  DCHECK(!wrap_called_) << "Clone() called after WrapIfBiggerThan()";
+#endif  // DCHECK_IS_ON()
+  *clone = DeserializeScriptValue(script_state, serialized_value_.Get(),
+                                  &blob_info_);
+}
+
+void IDBValueWrapper::WriteVarint(unsigned value, Vector<char>& output) {
+  // Writes an unsigned integer as a base-128 varint.
+  // The number is written, 7 bits at a time, from the least significant to
+  // the most significant 7 bits. Each byte, except the last, has the MSB set.
+  // See also https://developers.google.com/protocol-buffers/docs/encoding
+  do {
+    output.push_back((value & 0x7F) | 0x80);
+    value >>= 7;
+  } while (value);
+  output.back() &= 0x7F;
+}
+
+void IDBValueWrapper::WriteAsciiString(const String& value,
+                                       Vector<char>& output) {
+  DCHECK(value.Is8Bit() && value.ContainsOnlyASCII());
+
+  IDBValueWrapper::WriteVarint(value.length(), output);
+  output.Append(value.Characters8(), value.length());
+}
+
+bool IDBValueWrapper::WrapIfBiggerThan(unsigned max_bytes) {
+#if DCHECK_IS_ON()
+  DCHECK(!had_exception_) << __FUNCTION__
+                          << " called on wrapper with serialization exception";
+  DCHECK(!wrap_called_) << __FUNCTION__ << " called twice on the same wrapper";
+  wrap_called_ = true;
+#endif  // DCHECK_IS_ON()
+
+  serialized_value_->ToWireBytes(wire_bytes_);
+  if (wire_bytes_.size() <= max_bytes)
+    return false;
+
+  // TODO(pwnall): The MIME type should probably be an atomic string.
+  String mime_type(kWrapMimeType);
+  // TODO(crbug.com/721516): Use WebBlobRegistry::CreateBuilder instead of
+  //                         Blob::Create to avoid a buffer copy.
+  Blob* wrapper =
+      Blob::Create(reinterpret_cast<unsigned char*>(wire_bytes_.data()),
+                   wire_bytes_.size(), mime_type);
+
+  wrapper_handle_ = std::move(wrapper->GetBlobDataHandle());
+  blob_info_.emplace_back(wrapper_handle_->Uuid(), wrapper_handle_->GetType(),
+                          wrapper->size());
+
+  wire_bytes_.clear();
+
+  // Version 17 of SSV always writes a V8 envelope after the Blink envelope, so
+  // its output starts with 0xFF 0x11 0xFF. Therefore, we can use 0xFF 0x11 0xvv
+  // as an escape prefix, for 0x00 <= 0xvv < 0xFF.
+  wire_bytes_.push_back(kVersionTag);
+  wire_bytes_.push_back(kRequiresProcessingSSVPseudoVersion);
+  wire_bytes_.push_back(kBlobWrappedValue);
+  IDBValueWrapper::WriteVarint(wrapper->size(), wire_bytes_);
+  IDBValueWrapper::WriteAsciiString(wrapper->Uuid(), wire_bytes_);
+  return true;
+}
+
+void IDBValueWrapper::ExtractBlobDataHandles(
+    Vector<RefPtr<BlobDataHandle>>* blob_data_handles) {
+  for (const auto& kvp : serialized_value_->BlobDataHandles())
+    blob_data_handles->push_back(kvp.value);
+  if (wrapper_handle_)
+    blob_data_handles->push_back(std::move(wrapper_handle_));
+}
+
+RefPtr<SharedBuffer> IDBValueWrapper::ExtractWireBytes() {
+#if DCHECK_IS_ON()
+  DCHECK(!had_exception_) << __FUNCTION__
+                          << " called on wrapper with serialization exception";
+#endif  // DCHECK_IS_ON()
+
+  return SharedBuffer::AdoptVector(wire_bytes_);
+}
+
+IDBValueUnwrapper::IDBValueUnwrapper() {
+  Reset();
+}
+
+bool IDBValueUnwrapper::IsWrapped(IDBValue* value) {
+  DCHECK(value);
+
+  uint8_t header[3];
+  if (!value->data_ || value->data_->size() < sizeof(header))
+    return false;
+
+  value->data_->GetPartAsBytes(header, static_cast<size_t>(0), sizeof(header));
+  return header[0] == kVersionTag &&
+         header[1] == kRequiresProcessingSSVPseudoVersion &&
+         header[2] == kBlobWrappedValue;
+}
+
+RefPtr<IDBValue> IDBValueUnwrapper::CreateUnwrapped(
+    IDBValue* wrapped_value,
+    RefPtr<SharedBuffer>&& wrapper_blob_content) {
+  DCHECK(wrapped_value);
+  DCHECK(wrapped_value->data_);
+  DCHECK_GT(wrapped_value->blob_info_->size(), 0U);
+  DCHECK_EQ(wrapped_value->blob_info_->size(),
+            wrapped_value->blob_data_->size());
+
+  // Create an IDBValue with the same blob information, minus the last blob.
+  unsigned blob_count = wrapped_value->BlobInfo()->size() - 1;
+  std::unique_ptr<Vector<RefPtr<BlobDataHandle>>> blob_data =
+      WTF::MakeUnique<Vector<RefPtr<BlobDataHandle>>>();
+  blob_data->ReserveCapacity(blob_count);
+  std::unique_ptr<Vector<WebBlobInfo>> blob_info =
+      WTF::MakeUnique<Vector<WebBlobInfo>>();
+  blob_info->ReserveCapacity(blob_count);
+
+  for (unsigned i = 0; i < blob_count; ++i) {
+    blob_data->push_back((*wrapped_value->blob_data_)[i]);
+    blob_info->push_back((*wrapped_value->blob_info_)[i]);
+  }
+
+  return AdoptRef(new IDBValue(std::move(wrapper_blob_content),
+                               std::move(blob_data), std::move(blob_info)));
+}
+
+bool IDBValueUnwrapper::Parse(IDBValue* value) {
+  // Fast path that avoids unnecessary dynamic allocations.
+  if (!IDBValueUnwrapper::IsWrapped(value))
+    return false;
+
+  const uint8_t* data = reinterpret_cast<const uint8_t*>(value->data_->Data());
+  end_ = data + value->data_->size();
+  current_ = data + 3;
+
+  String blob_uuid;
+  if (!ReadVarint(blob_size_))
+    return Reset();
+  if (!ReadAsciiString(blob_uuid))
+    return Reset();
+
+  blob_handle_ = value->blob_data_->back();
+
+  // TODO(pwnall): Blobs seem to get different UUIDs when they're resurrected?

[jsbell, 2017/05/15 23:37:38]

Did you verify this or ... ?

[pwnall, 2017/05/19 18:27:35]

Done.
I just finished verifying this claim. Details in http://crbug.com/719007#c5

+  //               If this is true, stashing the UUID is useless.
+  if (blob_handle_->size() != blob_size_)
+    return Reset();
+
+  return true;
+}
+
+RefPtr<BlobDataHandle> IDBValueUnwrapper::WrapperBlobHandle() {
+  DCHECK(blob_handle_);
+
+  return std::move(blob_handle_);
+}
+
+bool IDBValueUnwrapper::ReadVarint(unsigned& value) {
+  value = 0;
+  unsigned shift = 0;
+  bool has_another_byte;
+  do {
+    if (current_ >= end_)
+      return false;
+
+    if (shift >= sizeof(unsigned) * 8)
+      return false;
+    uint8_t byte = *current_;
+    ++current_;
+    value |= static_cast<unsigned>(byte & 0x7F) << shift;
+    shift += 7;
+
+    has_another_byte = byte & 0x80;
+  } while (has_another_byte);
+  return true;
+}
+
+bool IDBValueUnwrapper::ReadAsciiString(String& value) {
+  unsigned length;
+  if (!ReadVarint(length))
+    return false;
+
+  DCHECK_LE(current_, end_);
+  if (end_ - current_ < static_cast<ptrdiff_t>(length))
+    return false;
+  String output(current_, length);
+  value.swap(output);
+  current_ += length;
+  return true;
+}
+
+}  // namespace blink