[BlobAsync] Moving async handling into BlobStorageContext & quota out.

storage/browser/blob/blob_storage_context.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_storage_context.h"
 
-#include <stddef.h>
-#include <stdint.h>
-
 #include <algorithm>
 #include <limits>
 #include <memory>
+#include <set>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram_macros.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "base/task_runner.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/trace_event/trace_event.h"
 #include "storage/browser/blob/blob_data_builder.h"
-#include "storage/browser/blob/blob_data_handle.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/blob_data_snapshot.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
 #include "url/gurl.h"
 
 namespace storage {
-using BlobRegistryEntry = BlobStorageRegistry::Entry;
-using BlobState = BlobStorageRegistry::BlobState;
-
-BlobStorageContext::BlobStorageContext() : memory_usage_(0) {}
+namespace {
+using ItemCopyEntry = BlobEntry::ItemCopyEntry;
+using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+
+bool IsBytes(DataElement::Type type) {
+  return type == DataElement::TYPE_BYTES ||
+         type == DataElement::TYPE_BYTES_DESCRIPTION;
+}
+}  // namespace
+
+BlobStorageContext::BlobFlattener::BlobFlattener(
+    const BlobDataBuilder& input_builder,
+    BlobEntry* output_blob,
+    BlobStorageRegistry* registry) {
+  const std::string& uuid = input_builder.uuid_;
+  std::set<std::string> dependent_blob_uuids;
+
+  size_t num_files_with_unknown_size = 0;
+  size_t num_building_dependent_blobs = 0;
+
+  base::CheckedNumeric<uint64_t> checked_total_size = 0;
+  base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0;
+
+  for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) {
+    const DataElement& input_element = input_item->data_element();
+    DataElement::Type type = input_element.type();
+    uint64_t length = input_element.length();
+
+    if (IsBytes(type)) {
+      DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length());

[kinuko, 2016/11/10 05:16:36]

Is this "0 +" for type conversion?

[dmurph, 2016/11/10 19:53:19]

I don't know why, but this make it link correctly.

+      checked_memory_quota_needed += length;
+      checked_total_size += length;
+      scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+          std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED);
+      pending_memory_items.push_back(item);
+      user_items.push_back(item.get());
+      output_blob->AppendSharedBlobItem(uuid, std::move(item));
+      continue;
+    }
+    if (type == DataElement::TYPE_BLOB) {
+      BlobEntry* ref_entry = registry->GetEntry(input_element.blob_uuid());
+
+      if (!ref_entry || input_element.blob_uuid() == uuid) {
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      if (BlobStatusIsError(ref_entry->status())) {
+        status = BlobStatus::ERR_REFERENCED_BLOB_BROKEN;
+        return;
+      }
+
+      if (ref_entry->total_size() == DataElement::kUnknownSize) {
+        // We can't reference a blob with unknown size.
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      if (dependent_blob_uuids.find(input_element.blob_uuid()) ==
+          dependent_blob_uuids.end()) {
+        dependent_blobs.push_back(
+            std::make_pair(input_element.blob_uuid(), ref_entry));
+        dependent_blob_uuids.insert(input_element.blob_uuid());
+        if (BlobStatusIsPending(ref_entry->status())) {
+          num_building_dependent_blobs++;
+        }
+      }
+
+      length = length == DataElement::kUnknownSize ? ref_entry->total_size()
+                                                   : input_element.length();
+      checked_total_size += length;
+
+      // If we're referencing the whole blob, then we don't need to slice.
+      if (input_element.offset() == 0 && length == ref_entry->total_size()) {
+        for (const auto& shareable_item : ref_entry->items()) {
+          output_blob->AppendSharedBlobItem(uuid, shareable_item);
+        }
+        continue;
+      }
+
+      // Validate our reference has good offset & length.
+      if (input_element.offset() + length > ref_entry->total_size()) {
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      BlobSlice slice(*ref_entry, input_element.offset(), length);
+
+      if (slice.first_source_item) {
+        copies.push_back(ItemCopyEntry(slice.first_source_item,
+                                       slice.first_item_slice_offset,
+                                       slice.dest_items.front()));
+        pending_memory_items.push_back(slice.dest_items.front());
+      }
+      if (slice.last_source_item) {
+        copies.push_back(
+            ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back()));
+        pending_memory_items.push_back(slice.dest_items.back());
+      }
+      checked_memory_quota_needed += slice.copying_memory_size;
+
+      for (auto& shareable_item : slice.dest_items) {
+        output_blob->AppendSharedBlobItem(uuid, std::move(shareable_item));
+      }
+      continue;
+    }
+
+    DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element))
+        << "File allocation not implemented.";
+    if (length == DataElement::kUnknownSize)
+      num_files_with_unknown_size++;
+
+    scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+        std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
+
+    checked_total_size += length;
+    output_blob->AppendSharedBlobItem(uuid, std::move(item));
+  }
+
+  if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) {
+    status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+    return;
+  }
+  if (!checked_total_size.IsValid() || !checked_memory_quota_needed.IsValid()) {
+    status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+    return;
+  }
+  total_size = checked_total_size.ValueOrDie();
+  memory_quota_needed = checked_memory_quota_needed.ValueOrDie();
+  if (memory_quota_needed) {
+    status = BlobStatus::PENDING_QUOTA;
+  } else {
+    status = BlobStatus::PENDING_INTERNALS;
+  }
+}
+
+BlobStorageContext::BlobFlattener::~BlobFlattener() {}
+
+BlobStorageContext::BlobSlice::BlobSlice(const BlobEntry& source,
+                                         uint64_t slice_offset,
+                                         uint64_t slice_size) {
+  const auto& source_items = source.items();
+  const auto& offsets = source.offsets();
+  DCHECK_LE(slice_offset + slice_size, source.total_size());
+  size_t item_index =
+      std::upper_bound(offsets.begin(), offsets.end(), slice_offset) -
+      offsets.begin();
+  uint64_t item_offset =
+      item_index == 0 ? slice_offset : slice_offset - offsets[item_index - 1];
+  size_t num_items = source_items.size();
+
+  size_t first_item_index = item_index;
+  copying_memory_size = 0;
+
+  // Read starting from 'first_item_index' and 'item_offset'.
+  for (uint64_t total_sliced = 0;
+       item_index < num_items && total_sliced < slice_size; item_index++) {
+    const scoped_refptr<BlobDataItem>& source_item =
+        source_items[item_index]->item();
+    uint64_t source_length = source_item->length();
+    DCHECK_NE(source_length, std::numeric_limits<uint64_t>::max());
+    DCHECK_NE(source_length, 0ull);
+
+    uint64_t read_size =
+        std::min(source_length - item_offset, slice_size - total_sliced);
+    total_sliced += read_size;
+
+    if (read_size == source_length) {
+      // We can share the entire item.
+      dest_items.push_back(source_items[item_index]);
+      continue;
+    }
+
+    scoped_refptr<BlobDataItem> data_item;
+    ShareableBlobDataItem::State state =
+        ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA;
+    switch (source_item->type()) {
+      case DataElement::TYPE_BYTES_DESCRIPTION:
+      case DataElement::TYPE_BYTES: {
+        if (item_index == first_item_index) {
+          first_item_slice_offset = item_offset;
+          first_source_item = source_items[item_index];
+        } else {
+          last_source_item = source_items[item_index];
+        }
+        copying_memory_size += read_size;
+        // Since we don't have quota yet for memory, we create temporary items
+        // for this data. When our blob is finished constructing, all dependent
+        // blobs are done, and we have enough memory quota, we'll copy the data
+        // over.
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToBytesDescription(base::checked_cast<size_t>(read_size));
+        data_item = new BlobDataItem(std::move(element));
+        state = ShareableBlobDataItem::QUOTA_NEEDED;
+        break;
+      }
+      case DataElement::TYPE_FILE: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToFilePathRange(
+            source_item->path(), source_item->offset() + item_offset, read_size,
+            source_item->expected_modification_time());
+        data_item =
+            new BlobDataItem(std::move(element), source_item->data_handle_);
+
+        DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element()))
+            << "File allocation unimplemented.";
+        break;
+      }
+      case DataElement::TYPE_FILE_FILESYSTEM: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToFileSystemUrlRange(
+            source_item->filesystem_url(), source_item->offset() + item_offset,
+            read_size, source_item->expected_modification_time());
+        data_item = new BlobDataItem(std::move(element));
+        break;
+      }
+      case DataElement::TYPE_DISK_CACHE_ENTRY: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToDiskCacheEntryRange(source_item->offset() + item_offset,
+                                          read_size);
+        data_item =
+            new BlobDataItem(std::move(element), source_item->data_handle_,
+                             source_item->disk_cache_entry(),
+                             source_item->disk_cache_stream_index(),
+                             source_item->disk_cache_side_stream_index());
+        break;
+      }
+      case DataElement::TYPE_BLOB:
+      case DataElement::TYPE_UNKNOWN:
+        CHECK(false) << "Illegal blob item type: " << source_item->type();
+    }
+    dest_items.push_back(
+        new ShareableBlobDataItem(std::move(data_item), state));
+    item_offset = 0;
+  }
+}
+
+BlobStorageContext::BlobSlice::~BlobSlice() {}
+
+BlobStorageContext::BlobStorageContext()
+    : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()),
+      ptr_factory_(this) {}
 
 BlobStorageContext::~BlobStorageContext() {
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID(
     const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  if (!entry)
     return nullptr;
-  }
-  return base::WrapUnique(
-      new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
-                         this, base::ThreadTaskRunnerHandle::Get().get()));
+  return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL(
     const GURL& url) {
   std::string uuid;
-  BlobRegistryEntry* entry = registry_.GetEntryFromURL(url, &uuid);
-  if (!entry) {
+  BlobEntry* entry = registry_.GetEntryFromURL(url, &uuid);
+  if (!entry)
     return nullptr;
-  }
-  return base::WrapUnique(
-      new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
-                         this, base::ThreadTaskRunnerHandle::Get().get()));
+  return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
     const BlobDataBuilder& external_builder) {
   TRACE_EVENT0("Blob", "Context::AddFinishedBlob");
-  CreatePendingBlob(external_builder.uuid(), external_builder.content_type_,
-                    external_builder.content_disposition_);
-  CompletePendingBlob(external_builder);
-  std::unique_ptr<BlobDataHandle> handle =
-      GetBlobDataFromUUID(external_builder.uuid_);
-  DecrementBlobRefCount(external_builder.uuid_);
-  return handle;
+  return BuildBlob(external_builder, TransportAllowedCallback());
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
     const BlobDataBuilder* builder) {
   DCHECK(builder);
   return AddFinishedBlob(*builder);
 }
 
+std::unique_ptr<BlobDataHandle> BlobStorageContext::AddBrokenBlob(
+    const std::string& uuid,
+    const std::string& content_type,
+    const std::string& content_disposition,
+    BlobStatus reason) {
+  DCHECK(!registry_.HasEntry(uuid));
+  DCHECK(BlobStatusIsError(reason));
+  BlobEntry* entry =
+      registry_.CreateEntry(uuid, content_type, content_disposition);
+  entry->status_ = reason;
+  FinishBuilding(entry);
+  return CreateHandle(uuid, entry);
+}
+
 bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url,
                                                const std::string& uuid) {
-  if (!registry_.CreateUrlMapping(blob_url, uuid)) {
+  if (!registry_.CreateUrlMapping(blob_url, uuid))
     return false;
-  }
   IncrementBlobRefCount(uuid);
   return true;
 }
 
 void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) {
   std::string uuid;
-  if (!registry_.DeleteURLMapping(blob_url, &uuid)) {
+  if (!registry_.DeleteURLMapping(blob_url, &uuid))
     return;
-  }
   DecrementBlobRefCount(uuid);
 }
 
-void BlobStorageContext::CreatePendingBlob(
-    const std::string& uuid,
-    const std::string& content_type,
-    const std::string& content_disposition) {
-  DCHECK(!registry_.GetEntry(uuid) && !uuid.empty());
-  registry_.CreateEntry(uuid, content_type, content_disposition);
+std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
+    const BlobDataBuilder& content,
+    const TransportAllowedCallback& can_populate_memory) {
+  DCHECK(!registry_.HasEntry(content.uuid_));
+
+  BlobEntry* entry = registry_.CreateEntry(content.uuid_, content.content_type_,
+                                           content.content_disposition_);
+
+  // This flattens all blob references in the transportion content out and
+  // stores the complete item representation in the internal data.
+  BlobFlattener flattener(content, entry, &registry_);
+
+  // If we contain invalid references we're invalid input.
+  if (flattener.status == BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS) {
+    CancelBuildingBlob(content.uuid_,
+                       BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  // If we contain broken references (or we reference ourself) we want to fail.
+  if (flattener.status == BlobStatus::ERR_REFERENCED_BLOB_BROKEN) {
+    CancelBuildingBlob(content.uuid_, BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) {
+    CancelBuildingBlob(content.uuid_, BlobStatus::ERR_OUT_OF_MEMORY);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  entry->status_ = flattener.status;
+  entry->size_ = flattener.total_size;
+  std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry);
+
+  size_t num_building_dependent_blobs = 0;
+  std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs;
+  // We hold a handle to all blobs we're using. This is important, as our memory
+  // accounting can be delayed until OnEnoughSizeForBlobData is called, and we
+  // only free memory on canceling when we've done this accounting. If a
+  // dependent blob is dereferenced, then we're the last blob holding onto that
+  // data item, and we need to account for that. So we prevent that case by
+  // holding onto all blobs.
+  for (const std::pair<std::string, BlobEntry*>& pending_blob :
+       flattener.dependent_blobs) {
+    dependent_blobs.push_back(
+        CreateHandle(pending_blob.first, pending_blob.second));
+    if (BlobStatusIsPending(pending_blob.second->status())) {
+      pending_blob.second->building_state_->build_completion_callbacks
+          .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished,
+                                ptr_factory_.GetWeakPtr(), content.uuid_));
+      num_building_dependent_blobs++;
+    }
+  }
+
+  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+         !can_populate_memory)
+      << "There is no pending content for the user to populate, so the "
+         "callback should be null.";
+  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+         can_populate_memory)
+      << "If we have pending content then there needs to be a callback "
+         "present.";
+
+  entry->building_state_.reset(new BlobEntry::BuildingState(
+      !flattener.user_items.empty(), can_populate_memory,
+      num_building_dependent_blobs, flattener.memory_quota_needed > 0));
+
+  BlobEntry::BuildingState* building_state = entry->building_state_.get();
+  std::swap(building_state->copies, flattener.copies);
+  std::swap(building_state->dependent_blobs, dependent_blobs);
+  std::swap(building_state->user_items, flattener.user_items);
+
+  if (flattener.memory_quota_needed > 0) {
+    // We can finish our blob before returning from this method.
+    base::WeakPtr<QuotaAllocationTask> pending_request =
+        memory_controller_.ReserveMemoryQuota(
+            std::move(flattener.pending_memory_items),
+            base::Bind(&BlobStorageContext::OnEnoughSizeForMemory,
+                       ptr_factory_.GetWeakPtr(), content.uuid_));
+    if (entry->building_state_)

[kinuko, 2016/11/10 05:16:36]

When and why this could be null?

[dmurph, 2016/11/10 19:53:19]

We can finish our blob before that method returns (see the above comment). Then
our building state will be null. I'll update the comment.

+      entry->building_state_->memory_quota_request = std::move(pending_request);
+  }
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+
+  return handle;
 }
 
-void BlobStorageContext::CompletePendingBlob(
-    const BlobDataBuilder& external_builder) {
-  BlobRegistryEntry* entry = registry_.GetEntry(external_builder.uuid());
+void BlobStorageContext::CancelBuildingBlob(const std::string& uuid,
+                                            BlobStatus reason) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  DCHECK(!entry->data.get()) << "Blob already constructed: "
-                             << external_builder.uuid();
-  // We want to handle storing our broken blob as well.
-  switch (entry->state) {
-    case BlobState::PENDING: {
-      entry->data_builder.reset(new InternalBlobData::Builder());
-      InternalBlobData::Builder* internal_data_builder =
-          entry->data_builder.get();
-
-      bool broken = false;
-      for (const auto& blob_item : external_builder.items_) {
-        IPCBlobCreationCancelCode error_code;
-        if (!AppendAllocatedBlobItem(external_builder.uuid_, blob_item,
-                                     internal_data_builder, &error_code)) {
-          broken = true;
-          memory_usage_ -= entry->data_builder->GetNonsharedMemoryUsage();
-          entry->state = BlobState::BROKEN;
-          entry->broken_reason = error_code;
-          entry->data_builder.reset(new InternalBlobData::Builder());
-          break;
-        }
-      }
-      entry->data = entry->data_builder->Build();
-      entry->data_builder.reset();
-      entry->state = broken ? BlobState::BROKEN : BlobState::COMPLETE;
-      break;
-    }
-    case BlobState::BROKEN: {
-      InternalBlobData::Builder builder;
-      entry->data = builder.Build();
-      break;
-    }
-    case BlobState::COMPLETE:
-      DCHECK(false) << "Blob already constructed: " << external_builder.uuid();
-      return;
+  DCHECK(BlobStatusIsError(reason));
+  TransportAllowedCallback user_data_population_callback;
+  if (entry->building_state_ &&
+      entry->building_state_->transport_allowed_callback) {
+    user_data_population_callback =
+        entry->building_state_->transport_allowed_callback;
+    entry->building_state_->transport_allowed_callback.Reset();
   }
-
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->data->items().size());
-  UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken",
-                        entry->state == BlobState::BROKEN);
-  if (entry->state == BlobState::BROKEN) {
-    UMA_HISTOGRAM_ENUMERATION(
-        "Storage.Blob.BrokenReason", static_cast<int>(entry->broken_reason),
-        (static_cast<int>(IPCBlobCreationCancelCode::LAST) + 1));
+  ClearAndFreeMemory(uuid, entry);
+  entry->status_ = reason;
+  if (user_data_population_callback) {
+    user_data_population_callback.Run(
+        reason, std::vector<BlobMemoryController::FileCreationInfo>());
   }
-  size_t total_memory = 0, nonshared_memory = 0;
-  entry->data->GetMemoryUsage(&total_memory, &nonshared_memory);
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", total_memory / 1024);
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
-                       nonshared_memory / 1024);
-  TRACE_COUNTER1("Blob", "MemoryStoreUsageBytes", memory_usage_);
-
-  auto runner = base::ThreadTaskRunnerHandle::Get();
-  for (const auto& callback : entry->build_completion_callbacks) {
-    runner->PostTask(FROM_HERE,
-                     base::Bind(callback, entry->state == BlobState::COMPLETE,
-                                entry->broken_reason));
-  }
-  entry->build_completion_callbacks.clear();
+  entry->items_.clear();
+  entry->offsets_.clear();
+  entry->size_ = 0;
+  FinishBuilding(entry);
 }
 
-void BlobStorageContext::CancelPendingBlob(const std::string& uuid,
-                                           IPCBlobCreationCancelCode reason) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  DCHECK(entry && entry->state == BlobState::PENDING);
-  entry->state = BlobState::BROKEN;
-  entry->broken_reason = reason;
-  CompletePendingBlob(BlobDataBuilder(uuid));
+void BlobStorageContext::NotifyTransportComplete(const std::string& uuid) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  CHECK(entry) << "There is no blob entry with uuid " << uuid;
+  DCHECK(BlobStatusIsPending(entry->status()));
+  NotifyTransportComplete(entry);
 }
 
 void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+  BlobEntry* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  ++(entry->refcount);
+  ++(entry->refcount_);
 }
 
 void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+  BlobEntry* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  DCHECK_GT(entry->refcount, 0u);
-  if (--(entry->refcount) == 0) {
-    size_t memory_freeing = 0;
-    if (entry->state == BlobState::COMPLETE) {
-      memory_freeing = entry->data->GetUnsharedMemoryUsage();
-      entry->data->RemoveBlobFromShareableItems(uuid);
-    }
-    DCHECK_LE(memory_freeing, memory_usage_);
-    memory_usage_ -= memory_freeing;
+  DCHECK_GT(entry->refcount_, 0u);
+  if (--(entry->refcount_) == 0) {
+    ClearAndFreeMemory(uuid, entry);
     registry_.DeleteEntry(uuid);
   }
 }
 
 std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot(
     const std::string& uuid) {
   std::unique_ptr<BlobDataSnapshot> result;
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (entry->state != BlobState::COMPLETE) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  if (entry->status() != BlobStatus::DONE)
     return result;
-  }
 
-  const InternalBlobData& data = *entry->data;
   std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot(
-      uuid, entry->content_type, entry->content_disposition));
-  snapshot->items_.reserve(data.items().size());
-  for (const auto& shareable_item : data.items()) {
+      uuid, entry->content_type(), entry->content_disposition()));
+  snapshot->items_.reserve(entry->items().size());
+  for (const auto& shareable_item : entry->items()) {
     snapshot->items_.push_back(shareable_item->item());
   }
+  memory_controller_.NotifyMemoryItemsUsed(entry->items());
   return snapshot;
 }
 
-bool BlobStorageContext::IsBroken(const std::string& uuid) const {
-  const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
-    return true;
-  }
-  return entry->state == BlobState::BROKEN;
-}
-
-bool BlobStorageContext::IsBeingBuilt(const std::string& uuid) const {
-  const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
-    return false;
-  }
-  return entry->state == BlobState::PENDING;
+BlobStatus BlobStorageContext::GetBlobStatus(const std::string& uuid) const {
+  const BlobEntry* entry = registry_.GetEntry(uuid);
+  if (!entry)
+    return BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+  return entry->status();
 }
 
 void BlobStorageContext::RunOnConstructionComplete(
     const std::string& uuid,
-    const BlobConstructedCallback& done) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+    const BlobStatusCallback& done) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  switch (entry->state) {
-    case BlobState::COMPLETE:
-      done.Run(true, IPCBlobCreationCancelCode::UNKNOWN);
-      return;
-    case BlobState::BROKEN:
-      done.Run(false, entry->broken_reason);
-      return;
-    case BlobState::PENDING:
-      entry->build_completion_callbacks.push_back(done);
-      return;
+  if (BlobStatusIsPending(entry->status())) {
+    entry->building_state_->build_completion_callbacks.push_back(done);
+    return;
   }
-  NOTREACHED();
+  done.Run(entry->status());
 }
 
-bool BlobStorageContext::AppendAllocatedBlobItem(
-    const std::string& target_blob_uuid,
-    scoped_refptr<BlobDataItem> blob_item,
-    InternalBlobData::Builder* target_blob_builder,
-    IPCBlobCreationCancelCode* error_code) {
-  DCHECK(error_code);
-  *error_code = IPCBlobCreationCancelCode::UNKNOWN;
-  bool error = false;
-
-  // The blob data is stored in the canonical way which only contains a
-  // list of Data, File, and FileSystem items. Aggregated TYPE_BLOB items
-  // are expanded into the primitive constituent types and reused if possible.
-  // 1) The Data item is denoted by the raw data and length.
-  // 2) The File item is denoted by the file path, the range and the expected
-  //    modification time.
-  // 3) The FileSystem File item is denoted by the FileSystem URL, the range
-  //    and the expected modification time.
-  // 4) The Blob item is denoted by the source blob and an offset and size.
-  //    Internal items that are fully used by the new blob (not cut by the
-  //    offset or size) are shared between the blobs.  Otherwise, the relevant
-  //    portion of the item is copied.
-
-  DCHECK(blob_item->data_element_ptr());
-  const DataElement& data_element = blob_item->data_element();
-  uint64_t length = data_element.length();
-  uint64_t offset = data_element.offset();
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
-                       memory_usage_ / 1024);
-  switch (data_element.type()) {
-    case DataElement::TYPE_BYTES:
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024);
-      DCHECK(!offset);
-      if (memory_usage_ + length > kBlobStorageMaxMemoryUsage) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
-        break;
-      }
-      memory_usage_ += length;
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITH_QUOTA));
-      break;
-    case DataElement::TYPE_FILE: {
-      bool full_file = (length == std::numeric_limits<uint64_t>::max());
-      UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file);
-      if (!full_file) {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File",
-                             (length - offset) / 1024);
-      }
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_FILE_FILESYSTEM: {
-      bool full_file = (length == std::numeric_limits<uint64_t>::max());
-      UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown",
-                            full_file);
-      if (!full_file) {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem",
-                             (length - offset) / 1024);
-      }
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_BLOB: {
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob",
-                           (length - offset) / 1024);
-      // We grab the handle to ensure it stays around while we copy it.
-      std::unique_ptr<BlobDataHandle> src =
-          GetBlobDataFromUUID(data_element.blob_uuid());
-      if (!src || src->IsBroken() || src->IsBeingBuilt()) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN;
-        break;
-      }
-      BlobRegistryEntry* other_entry =
-          registry_.GetEntry(data_element.blob_uuid());
-      DCHECK(other_entry->data);
-      if (!AppendBlob(target_blob_uuid, *other_entry->data, offset, length,
-                      target_blob_builder)) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
-      }
-      break;
-    }
-    case DataElement::TYPE_DISK_CACHE_ENTRY: {
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry",
-                           (length - offset) / 1024);
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_BYTES_DESCRIPTION:
-    case DataElement::TYPE_UNKNOWN:
-      NOTREACHED();
-      break;
-  }
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend",
-                       memory_usage_ / 1024);
-  return !error;
+std::unique_ptr<BlobDataHandle> BlobStorageContext::CreateHandle(
+    const std::string& uuid,
+    BlobEntry* entry) {
+  return base::WrapUnique(new BlobDataHandle(
+      uuid, entry->content_type_, entry->content_disposition_, entry->size_,
+      this, base::ThreadTaskRunnerHandle::Get().get()));
 }
 
-bool BlobStorageContext::AppendBlob(
-    const std::string& target_blob_uuid,
-    const InternalBlobData& blob,
-    uint64_t offset,
-    uint64_t length,
-    InternalBlobData::Builder* target_blob_builder) {
-  DCHECK_GT(length, 0ull);
+void BlobStorageContext::NotifyTransportComplete(BlobEntry* entry) {
+  DCHECK(entry);
+  SetItemStateToPopulated(&entry->building_state_->user_items);
+  entry->status_ = BlobStatus::PENDING_INTERNALS;
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
 
-  const std::vector<scoped_refptr<ShareableBlobDataItem>>& items = blob.items();
-  auto iter = items.begin();
-  if (offset) {
-    for (; iter != items.end(); ++iter) {
-      const BlobDataItem& item = *(iter->get()->item());
-      if (offset >= item.length())
-        offset -= item.length();
-      else
-        break;
+void BlobStorageContext::FinishBuilding(BlobEntry* entry) {
+  DCHECK(entry);
+
+  if (BlobStatusIsPending(entry->status_)) {

[kinuko, 2016/11/10 05:16:36]

Is PENDING_INTERNALS used mostly for this if block?  Setting entry state to
PENDING when we're done feels a bit weird.

[dmurph, 2016/11/10 19:53:19]

Yes. We're still not done if we haven't copied over all of the dependent blob
items.

PENDING_INTERNALS is also used for us waiting for dependent blobs. Those
status's don't have a super great 1:1 mapping of our internals, as the dependent
blob stuff can be happening during quota request & transportation. The
constraints are:
1. We need quota before transportation
2. We need quota before copying memory from dependent blobs.

When we introduce disk, it gets more complicated as we CAN be requesting one
quota type (file) for transportation and another (memory) for copies. So a
linear state system doesn't really work later. But it mostly works here.

+    for (const ItemCopyEntry& copy : entry->building_state_->copies) {
+      // Our source item can be a file if it was a slice of an unpopulated file,
+      // or a slice of data that was then paged to disk.
+      size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length());
+      DataElement::Type dest_type = copy.dest_item->item()->type();
+      switch (copy.source_item->item()->type()) {
+        case DataElement::TYPE_BYTES: {
+          DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION);
+          const char* src_data =
+              copy.source_item->item()->bytes() + copy.source_item_offset;
+          copy.dest_item->item()->item_->SetToBytes(src_data, dest_size);
+        } break;
+        case DataElement::TYPE_FILE:
+        case DataElement::TYPE_UNKNOWN:
+        case DataElement::TYPE_BLOB:
+        case DataElement::TYPE_BYTES_DESCRIPTION:
+        case DataElement::TYPE_FILE_FILESYSTEM:
+        case DataElement::TYPE_DISK_CACHE_ENTRY:
+          NOTREACHED();
+          break;
+      }
+      copy.dest_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
+    }
+    entry->status_ = BlobStatus::DONE;
+  }
+
+  std::vector<BlobStatusCallback> callbacks;
+  if (entry->building_state_.get()) {
+    std::swap(callbacks, entry->building_state_->build_completion_callbacks);
+    entry->building_state_.reset();
+  }
+
+  memory_controller_.NotifyMemoryItemsUsed(entry->items());
+
+  auto runner = base::ThreadTaskRunnerHandle::Get();
+  for (const auto& callback : callbacks)
+    runner->PostTask(FROM_HERE, base::Bind(callback, entry->status()));
+
+  for (const auto& shareable_item : entry->items()) {
+    DCHECK_NE(DataElement::TYPE_BYTES_DESCRIPTION,
+              shareable_item->item()->type());
+    DCHECK(shareable_item->IsPopulated()) << shareable_item->state();
+  }
+}
+
+void BlobStorageContext::RequestTransport(
+    BlobEntry* entry,
+    std::vector<BlobMemoryController::FileCreationInfo> files) {
+  BlobEntry::BuildingState* building_state = entry->building_state_.get();
+  if (building_state->transport_allowed_callback) {
+    TransportAllowedCallback user_data_population_callback =
+        building_state->transport_allowed_callback;
+    building_state->transport_allowed_callback.Reset();
+    user_data_population_callback.Run(BlobStatus::PENDING_TRANSPORT,
+                                      std::move(files));
+    return;
+  }
+  DCHECK(files.empty());
+  NotifyTransportComplete(entry);
+}
+
+void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid,
+                                               bool success) {
+  if (!success) {
+    CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+    return;
+  }
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  if (!entry || !entry->building_state_.get())
+    return;
+  BlobEntry::BuildingState& building_state = *entry->building_state_;
+  DCHECK(!building_state.memory_quota_request);
+
+  if (building_state.transport_items_present) {
+    entry->status_ = BlobStatus::PENDING_TRANSPORT;
+    RequestTransport(entry,
+                     std::vector<BlobMemoryController::FileCreationInfo>());
+  } else {
+    entry->status_ = BlobStatus::PENDING_INTERNALS;
+  }
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::OnDependentBlobFinished(
+    const std::string& owning_blob_uuid,
+    BlobStatus status) {
+  BlobEntry* entry = registry_.GetEntry(owning_blob_uuid);
+  if (!entry || !entry->building_state_)
+    return;
+
+  if (BlobStatusIsError(status)) {
+    DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status)
+        << "Referenced blob should never be dereferenced while we "
+        << "are depending on it, as our system holds a handle.";
+    CancelBuildingBlob(owning_blob_uuid,
+                       BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+    return;
+  }
+  DCHECK_GT(entry->building_state_->num_building_dependent_blobs, 0u);
+  --entry->building_state_->num_building_dependent_blobs;
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::ClearAndFreeMemory(const std::string& uuid,
+                                            BlobEntry* entry) {
+  if (entry->building_state_) {
+    BlobEntry::BuildingState* building_state = entry->building_state_.get();
+    if (building_state->memory_quota_request) {
+      building_state->memory_quota_request->Cancel();
     }
   }
+  entry->RemoveBlobFromShareableItems(uuid);
+  entry->items_.clear();
+  entry->offsets_.clear();
+}
 
-  for (; iter != items.end() && length > 0; ++iter) {
-    scoped_refptr<ShareableBlobDataItem> shareable_item = iter->get();
-    const BlobDataItem& item = *(shareable_item->item());
-    uint64_t item_length = item.length();
-    DCHECK_GT(item_length, offset);
-    uint64_t current_length = item_length - offset;
-    uint64_t new_length = current_length > length ? length : current_length;
-
-    bool reusing_blob_item = offset == 0 && new_length == item.length();
-    UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item);
-    if (reusing_blob_item) {
-      shareable_item->referencing_blobs_mutable()->insert(target_blob_uuid);
-      target_blob_builder->AppendSharedBlobItem(shareable_item);
-      length -= new_length;
-      continue;
-    }
-
-    // We need to do copying of the items when we have a different offset or
-    // length
-    switch (item.type()) {
-      case DataElement::TYPE_BYTES: {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes",
-                             new_length / 1024);
-        if (memory_usage_ + new_length > kBlobStorageMaxMemoryUsage) {
-          return false;
-        }
-        DCHECK(!item.offset());
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToBytes(item.bytes() + offset,
-                            static_cast<int64_t>(new_length));
-        memory_usage_ += new_length;
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid, new BlobDataItem(std::move(element)),
-            ShareableBlobDataItem::POPULATED_WITH_QUOTA));
-      } break;
-      case DataElement::TYPE_FILE: {
-        DCHECK_NE(item.length(), std::numeric_limits<uint64_t>::max())
-            << "We cannot use a section of a file with an unknown length";
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File",
-                             new_length / 1024);
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToFilePathRange(item.path(), item.offset() + offset,
-                                    new_length,
-                                    item.expected_modification_time());
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid,
-            new BlobDataItem(std::move(element), item.data_handle_),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_FILE_FILESYSTEM: {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
-                             new_length / 1024);
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToFileSystemUrlRange(item.filesystem_url(),
-                                         item.offset() + offset, new_length,
-                                         item.expected_modification_time());
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid, new BlobDataItem(std::move(element)),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_DISK_CACHE_ENTRY: {
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToDiskCacheEntryRange(item.offset() + offset,
-                                          new_length);
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid,
-            new BlobDataItem(std::move(element), item.data_handle_,
-                             item.disk_cache_entry(),
-                             item.disk_cache_stream_index(),
-                             item.disk_cache_side_stream_index()),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_BYTES_DESCRIPTION:
-      case DataElement::TYPE_BLOB:
-      case DataElement::TYPE_UNKNOWN:
-        CHECK(false) << "Illegal blob item type: " << item.type();
-    }
-    length -= new_length;
-    offset = 0;
+void BlobStorageContext::SetItemStateToPopulated(
+    std::vector<ShareableBlobDataItem*>* items) {
+  for (ShareableBlobDataItem* shareable_item : *items) {

[kinuko, 2016/11/10 05:16:36]

nit: I'd probably just put this 4-lines loop directly in NotifyTransportComplete

[dmurph, 2016/11/10 19:53:19]

Done.

+    DCHECK(shareable_item->state() == ShareableBlobDataItem::QUOTA_GRANTED);
+    shareable_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
   }
-  return true;
 }
 
 }  // namespace storage