[BlobAsync] Disk support for blob storage

storage/browser/blob/blob_flattener.cc

Index: storage/browser/blob/blob_flattener.cc
diff --git a/storage/browser/blob/blob_flattener.cc b/storage/browser/blob/blob_flattener.cc
new file mode 100644
index 0000000000000000000000000000000000000000..70358829ee293a60b03af528b347748868293700
--- /dev/null
+++ b/storage/browser/blob/blob_flattener.cc
@@ -0,0 +1,274 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "storage/browser/blob/blob_flattener.h"
+
+#include <algorithm>
+#include <limits>
+#include <set>
+#include <utility>
+
+#include "storage/browser/blob/blob_async_transport_request_builder.h"
+#include "storage/browser/blob/blob_data_builder.h"
+#include "storage/browser/blob/blob_data_item.h"
+#include "storage/browser/blob/blob_storage_context.h"
+#include "storage/browser/blob/blob_storage_registry.h"
+#include "storage/browser/blob/internal_blob_data.h"
+#include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
+
+namespace storage {
+namespace {
+using ItemCopyEntry = InternalBlobData::ItemCopyEntry;
+
+bool IsBytes(DataElement::Type type) {
+  return type == DataElement::TYPE_BYTES ||
+         type == DataElement::TYPE_BYTES_DESCRIPTION;
+}
+}  // namespace
+
+// static
+BlobFlattener::BlobFlattener(const BlobDataBuilder& input_builder,
+                             InternalBlobData* output_blob,
+                             BlobStorageRegistry* registry) {
+  const std::string& uuid = input_builder.uuid_;
+  std::set<std::string> dependent_blob_uuids;
+  bool contains_pending_content = false;
+
+  size_t num_files_with_unknown_size = 0;
+
+  for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) {
+    const DataElement& input_element = input_item->data_element();
+    DataElement::Type type = input_element.type();
+    uint64_t length = input_element.length();
+    contains_pending_content |= type == DataElement::TYPE_BYTES_DESCRIPTION;
+
+    if (IsBytes(type)) {
+      DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length());
+      transport_quota_needed += length;
+      total_size += length;
+      scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+          std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED);
+      transport_pending_items.push_back(item.get());
+      output_blob->AppendSharedBlobItem(uuid, std::move(item));
+      continue;
+    }
+    if (type == DataElement::TYPE_BLOB) {
+      InternalBlobData* ref_entry =
+          registry->GetEntry(input_element.blob_uuid());
+
+      if (!ref_entry || input_element.blob_uuid() == uuid) {
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      if (BlobStatusIsError(ref_entry->status())) {
+        status = BlobStatus::ERR_REFERENCED_BLOB_BROKEN;
+        return;
+      }
+
+      if (ref_entry->total_size() == DataElement::kUnknownSize) {
+        // We can't reference a blob with unknown size.
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      if (dependent_blob_uuids.find(input_element.blob_uuid()) ==
+          dependent_blob_uuids.end()) {
+        dependent_blobs.push_back(
+            std::make_pair(input_element.blob_uuid(), ref_entry));
+        dependent_blob_uuids.insert(input_element.blob_uuid());
+      }
+
+      length = length == DataElement::kUnknownSize ? ref_entry->total_size()
+                                                   : input_element.length();
+      total_size += length;
+
+      // If we're referencing the whole blob, then we don't need to slice.
+      if (input_element.offset() == 0 && length == ref_entry->total_size()) {
+        for (const auto& shareable_item : ref_entry->items()) {
+          output_blob->AppendSharedBlobItem(uuid, shareable_item);
+        }
+        continue;
+      }
+
+      // Validate our reference has good offset & length.
+      if (input_element.offset() + input_element.length() >
+          ref_entry->total_size()) {
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      BlobSlice slice(*ref_entry, input_element.offset(),
+                      input_element.length());

[Marijn Kruisselbrink, 2016/08/05 23:23:54]

Should this use input_element.length() (which can be kUnknownSize) or length
(which is guaranteed at this point to have a known value, and usually is the
same as input_element.length())?

[dmurph, 2016/08/19 00:18:32]

Good idea. Thanks.

+
+      if (slice.first_source_item) {
+        copies.push_back(ItemCopyEntry(slice.first_source_item,
+                                       slice.first_item_slice_offset,
+                                       slice.dest_items.front()));
+        copy_pending_items.push_back(slice.dest_items.front().get());
+      }
+      if (slice.last_source_item) {
+        copies.push_back(
+            ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back()));
+        copy_pending_items.push_back(slice.dest_items.back().get());
+      }
+      copy_quota_needed += slice.copying_memory_size;
+
+      for (auto& shareable_item : slice.dest_items) {
+        output_blob->AppendSharedBlobItem(uuid, std::move(shareable_item));
+      }
+      continue;
+    }
+
+    // If the source item is a temporary file item, then we need to keep track
+    // of that and mark is as needing quota.
+    scoped_refptr<ShareableBlobDataItem> item;
+    if (type == DataElement::TYPE_FILE &&
+        BlobDataBuilder::IsTemporaryFileItem(input_element)) {
+      item = new ShareableBlobDataItem(std::move(input_item),
+                                       ShareableBlobDataItem::QUOTA_NEEDED);
+      contains_pending_content = true;
+      transport_pending_items.push_back(item.get());
+      transport_quota_needed += length;
+    } else {
+      item = new ShareableBlobDataItem(
+          std::move(input_item),
+          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
+    }
+    if (length == DataElement::kUnknownSize)
+      num_files_with_unknown_size++;
+
+    total_size += length;

[Marijn Kruisselbrink, 2016/08/05 23:23:54]

It seems a bit odd to sometimes be adding kUnknownSize to total_size (and above
to transport_quote_needed). I suppose since we'll error out later if that
happened with more than one total input item it all works out fine...

[dmurph, 2016/08/19 00:18:32]

Yeah, since total_size is a safe math variable anyways, I think this should be
fine. We'll error out, or have an invalid total size, both of which are caught.

+    output_blob->AppendSharedBlobItem(uuid, std::move(item));
+  }
+
+  if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) {
+    LOG(ERROR) << "We only allow an unknown size when it's a single file item";
+    status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+    return;
+  }
+  status = contains_pending_content ? BlobStatus::PENDING : BlobStatus::DONE;

[Marijn Kruisselbrink, 2016/08/05 23:23:54]

Currently you have whoever called BlobFlattener check for IsValid on the various
checked numbers. Would it make sense to move those checks to here and set status
appropriately, thereby making calling code simpler?

[dmurph, 2016/08/19 00:18:33]

Done.

+}
+
+BlobFlattener::~BlobFlattener() {}
+
+BlobSlice::BlobSlice(const InternalBlobData& source,
+                     uint64_t slice_offset,
+                     uint64_t slice_size) {
+  const auto& source_items = source.items();
+  const auto& offsets = source.offsets();
+  LOG(ERROR) << "doing a slice at " << slice_offset << " with size "
+             << slice_size;
+  DCHECK_LE(slice_offset + slice_size, source.total_size());
+  size_t item_index =
+      std::upper_bound(offsets.begin(), offsets.end(), slice_offset) -
+      offsets.begin();
+  uint64_t item_offset =
+      item_index == 0 ? slice_offset : slice_offset - offsets[item_index - 1];
+  size_t num_items = source_items.size();
+
+  size_t first_item_index = item_index;
+  copying_memory_size = 0;
+
+  // Read starting from 'first_item_index' and 'item_offset'.
+  for (uint64_t total_sliced = 0;
+       item_index < num_items && total_sliced < slice_size; item_index++) {
+    const scoped_refptr<BlobDataItem>& source_item =
+        source_items[item_index]->item();
+    uint64_t source_length = source_item->length();
+    DCHECK_NE(source_length, std::numeric_limits<uint64_t>::max());
+    DCHECK_NE(source_length, 0ull);
+
+    uint64_t read_size =
+        std::min(source_length - item_offset, slice_size - total_sliced);
+    total_sliced += read_size;
+
+    if (read_size == source_length) {
+      // We can share the entire item.
+      LOG(ERROR) << "we can share";
+      dest_items.push_back(source_items[item_index]);
+      continue;
+    }
+
+    scoped_refptr<BlobDataItem> data_item;
+    ShareableBlobDataItem::State state =
+        ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA;
+    switch (source_item->type()) {
+      case DataElement::TYPE_BYTES_DESCRIPTION:
+      case DataElement::TYPE_BYTES: {
+        if (item_index == first_item_index) {
+          first_item_slice_offset = item_offset;
+          first_source_item = source_items[item_index];
+        } else {
+          last_source_item = source_items[item_index];
+        }
+        LOG(ERROR) << "we're slicing a bytes item!";
+        copying_memory_size += read_size;
+        // Since we don't have quota yet for memory, we create temporary items
+        // for this data. When our blob is finished constructing, all dependent
+        // blobs are done, and we have enough memory quota, we'll copy the data
+        // over.
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToBytesDescription(base::checked_cast<size_t>(read_size));
+        data_item = new BlobDataItem(std::move(element));
+        state = ShareableBlobDataItem::QUOTA_NEEDED;
+        break;
+      }
+      case DataElement::TYPE_FILE: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToFilePathRange(
+            source_item->path(), source_item->offset() + item_offset, read_size,
+            source_item->expected_modification_time());
+        data_item =
+            new BlobDataItem(std::move(element), source_item->data_handle_);
+
+        if (BlobDataBuilder::IsTemporaryFileItem(source_item->data_element())) {
+          // Since we don't have file path / reference for our future file, we
+          // create another file item with this temporary file name. When our
+          // blob is finished constructing, all dependent blobs are done, and we
+          // can copy the handle over.
+          LOG(ERROR) << "we're slicing a temp file item!";
+          if (item_index == first_item_index) {
+            first_item_slice_offset = item_offset;
+            first_source_item = source_items[item_index];
+          } else {
+            last_source_item = source_items[item_index];
+          }
+          state = ShareableBlobDataItem::QUOTA_NEEDED;
+        }
+        break;
+      }
+      case DataElement::TYPE_FILE_FILESYSTEM: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToFileSystemUrlRange(
+            source_item->filesystem_url(), source_item->offset() + item_offset,
+            read_size, source_item->expected_modification_time());
+        data_item = new BlobDataItem(std::move(element));
+        break;
+      }
+      case DataElement::TYPE_DISK_CACHE_ENTRY: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToDiskCacheEntryRange(source_item->offset() + item_offset,
+                                          read_size);
+        data_item =
+            new BlobDataItem(std::move(element), source_item->data_handle_,
+                             source_item->disk_cache_entry(),
+                             source_item->disk_cache_stream_index(),
+                             source_item->disk_cache_side_stream_index());
+        break;
+      }
+      case DataElement::TYPE_BLOB:
+      case DataElement::TYPE_UNKNOWN:
+        CHECK(false) << "Illegal blob item type: " << source_item->type();
+    }
+    dest_items.push_back(
+        new ShareableBlobDataItem(std::move(data_item), state));
+    item_offset = 0;
+  }
+}
+
+BlobSlice::~BlobSlice() {}
+
+}  // namespace storage