[BlobAsync] Moving async handling into BlobStorageContext & quota out.

storage/browser/blob/blob_storage_context.cc

Index: storage/browser/blob/blob_storage_context.cc
diff --git a/storage/browser/blob/blob_storage_context.cc b/storage/browser/blob/blob_storage_context.cc
index fb1bc5b9f0d29a7a32a11c3d19b8e764d89eb3fb..a5ece241e32e38023368d55a700e490f92fa35fb 100644
--- a/storage/browser/blob/blob_storage_context.cc
+++ b/storage/browser/blob/blob_storage_context.cc
@@ -4,12 +4,10 @@
 
 #include "storage/browser/blob/blob_storage_context.h"
 
-#include <stddef.h>
-#include <stdint.h>
-
 #include <algorithm>
 #include <limits>
 #include <memory>
+#include <set>
 #include <utility>
 
 #include "base/bind.h"
@@ -19,57 +17,286 @@
 #include "base/memory/ptr_util.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram_macros.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "base/task_runner.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/trace_event/trace_event.h"
 #include "storage/browser/blob/blob_data_builder.h"
-#include "storage/browser/blob/blob_data_handle.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/blob_data_snapshot.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
 #include "url/gurl.h"
 
 namespace storage {
-using BlobRegistryEntry = BlobStorageRegistry::Entry;
-using BlobState = BlobStorageRegistry::BlobState;
+namespace {
+using ItemCopyEntry = BlobEntry::ItemCopyEntry;
+using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+
+bool IsBytes(DataElement::Type type) {
+  return type == DataElement::TYPE_BYTES ||
+         type == DataElement::TYPE_BYTES_DESCRIPTION;
+}
+}  // namespace
+
+BlobStorageContext::BlobFlattener::BlobFlattener(
+    const BlobDataBuilder& input_builder,
+    BlobEntry* output_blob,
+    BlobStorageRegistry* registry) {
+  const std::string& uuid = input_builder.uuid_;
+  std::set<std::string> dependent_blob_uuids;
+
+  size_t num_files_with_unknown_size = 0;
+  size_t num_building_dependent_blobs = 0;
+
+  base::CheckedNumeric<uint64_t> checked_total_size = 0;
+  base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0;
+
+  for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) {
+    const DataElement& input_element = input_item->data_element();
+    DataElement::Type type = input_element.type();
+    uint64_t length = input_element.length();
+
+    if (IsBytes(type)) {
+      DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length());

[kinuko, 2016/11/10 05:16:36]

Is this "0 +" for type conversion?

[dmurph, 2016/11/10 19:53:19]

I don't know why, but this make it link correctly.

+      checked_memory_quota_needed += length;
+      checked_total_size += length;
+      scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+          std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED);
+      pending_memory_items.push_back(item);
+      user_items.push_back(item.get());
+      output_blob->AppendSharedBlobItem(uuid, std::move(item));
+      continue;
+    }
+    if (type == DataElement::TYPE_BLOB) {
+      BlobEntry* ref_entry = registry->GetEntry(input_element.blob_uuid());
+
+      if (!ref_entry || input_element.blob_uuid() == uuid) {
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      if (BlobStatusIsError(ref_entry->status())) {
+        status = BlobStatus::ERR_REFERENCED_BLOB_BROKEN;
+        return;
+      }
+
+      if (ref_entry->total_size() == DataElement::kUnknownSize) {
+        // We can't reference a blob with unknown size.
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      if (dependent_blob_uuids.find(input_element.blob_uuid()) ==
+          dependent_blob_uuids.end()) {
+        dependent_blobs.push_back(
+            std::make_pair(input_element.blob_uuid(), ref_entry));
+        dependent_blob_uuids.insert(input_element.blob_uuid());
+        if (BlobStatusIsPending(ref_entry->status())) {
+          num_building_dependent_blobs++;
+        }
+      }
+
+      length = length == DataElement::kUnknownSize ? ref_entry->total_size()
+                                                   : input_element.length();
+      checked_total_size += length;
+
+      // If we're referencing the whole blob, then we don't need to slice.
+      if (input_element.offset() == 0 && length == ref_entry->total_size()) {
+        for (const auto& shareable_item : ref_entry->items()) {
+          output_blob->AppendSharedBlobItem(uuid, shareable_item);
+        }
+        continue;
+      }
+
+      // Validate our reference has good offset & length.
+      if (input_element.offset() + length > ref_entry->total_size()) {
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      BlobSlice slice(*ref_entry, input_element.offset(), length);
 
-BlobStorageContext::BlobStorageContext() : memory_usage_(0) {}
+      if (slice.first_source_item) {
+        copies.push_back(ItemCopyEntry(slice.first_source_item,
+                                       slice.first_item_slice_offset,
+                                       slice.dest_items.front()));
+        pending_memory_items.push_back(slice.dest_items.front());
+      }
+      if (slice.last_source_item) {
+        copies.push_back(
+            ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back()));
+        pending_memory_items.push_back(slice.dest_items.back());
+      }
+      checked_memory_quota_needed += slice.copying_memory_size;
+
+      for (auto& shareable_item : slice.dest_items) {
+        output_blob->AppendSharedBlobItem(uuid, std::move(shareable_item));
+      }
+      continue;
+    }
+
+    DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element))
+        << "File allocation not implemented.";
+    if (length == DataElement::kUnknownSize)
+      num_files_with_unknown_size++;
+
+    scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+        std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
+
+    checked_total_size += length;
+    output_blob->AppendSharedBlobItem(uuid, std::move(item));
+  }
+
+  if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) {
+    status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+    return;
+  }
+  if (!checked_total_size.IsValid() || !checked_memory_quota_needed.IsValid()) {
+    status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+    return;
+  }
+  total_size = checked_total_size.ValueOrDie();
+  memory_quota_needed = checked_memory_quota_needed.ValueOrDie();
+  if (memory_quota_needed) {
+    status = BlobStatus::PENDING_QUOTA;
+  } else {
+    status = BlobStatus::PENDING_INTERNALS;
+  }
+}
+
+BlobStorageContext::BlobFlattener::~BlobFlattener() {}
+
+BlobStorageContext::BlobSlice::BlobSlice(const BlobEntry& source,
+                                         uint64_t slice_offset,
+                                         uint64_t slice_size) {
+  const auto& source_items = source.items();
+  const auto& offsets = source.offsets();
+  DCHECK_LE(slice_offset + slice_size, source.total_size());
+  size_t item_index =
+      std::upper_bound(offsets.begin(), offsets.end(), slice_offset) -
+      offsets.begin();
+  uint64_t item_offset =
+      item_index == 0 ? slice_offset : slice_offset - offsets[item_index - 1];
+  size_t num_items = source_items.size();
+
+  size_t first_item_index = item_index;
+  copying_memory_size = 0;
+
+  // Read starting from 'first_item_index' and 'item_offset'.
+  for (uint64_t total_sliced = 0;
+       item_index < num_items && total_sliced < slice_size; item_index++) {
+    const scoped_refptr<BlobDataItem>& source_item =
+        source_items[item_index]->item();
+    uint64_t source_length = source_item->length();
+    DCHECK_NE(source_length, std::numeric_limits<uint64_t>::max());
+    DCHECK_NE(source_length, 0ull);
+
+    uint64_t read_size =
+        std::min(source_length - item_offset, slice_size - total_sliced);
+    total_sliced += read_size;
+
+    if (read_size == source_length) {
+      // We can share the entire item.
+      dest_items.push_back(source_items[item_index]);
+      continue;
+    }
+
+    scoped_refptr<BlobDataItem> data_item;
+    ShareableBlobDataItem::State state =
+        ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA;
+    switch (source_item->type()) {
+      case DataElement::TYPE_BYTES_DESCRIPTION:
+      case DataElement::TYPE_BYTES: {
+        if (item_index == first_item_index) {
+          first_item_slice_offset = item_offset;
+          first_source_item = source_items[item_index];
+        } else {
+          last_source_item = source_items[item_index];
+        }
+        copying_memory_size += read_size;
+        // Since we don't have quota yet for memory, we create temporary items
+        // for this data. When our blob is finished constructing, all dependent
+        // blobs are done, and we have enough memory quota, we'll copy the data
+        // over.
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToBytesDescription(base::checked_cast<size_t>(read_size));
+        data_item = new BlobDataItem(std::move(element));
+        state = ShareableBlobDataItem::QUOTA_NEEDED;
+        break;
+      }
+      case DataElement::TYPE_FILE: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToFilePathRange(
+            source_item->path(), source_item->offset() + item_offset, read_size,
+            source_item->expected_modification_time());
+        data_item =
+            new BlobDataItem(std::move(element), source_item->data_handle_);
+
+        DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element()))
+            << "File allocation unimplemented.";
+        break;
+      }
+      case DataElement::TYPE_FILE_FILESYSTEM: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToFileSystemUrlRange(
+            source_item->filesystem_url(), source_item->offset() + item_offset,
+            read_size, source_item->expected_modification_time());
+        data_item = new BlobDataItem(std::move(element));
+        break;
+      }
+      case DataElement::TYPE_DISK_CACHE_ENTRY: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToDiskCacheEntryRange(source_item->offset() + item_offset,
+                                          read_size);
+        data_item =
+            new BlobDataItem(std::move(element), source_item->data_handle_,
+                             source_item->disk_cache_entry(),
+                             source_item->disk_cache_stream_index(),
+                             source_item->disk_cache_side_stream_index());
+        break;
+      }
+      case DataElement::TYPE_BLOB:
+      case DataElement::TYPE_UNKNOWN:
+        CHECK(false) << "Illegal blob item type: " << source_item->type();
+    }
+    dest_items.push_back(
+        new ShareableBlobDataItem(std::move(data_item), state));
+    item_offset = 0;
+  }
+}
+
+BlobStorageContext::BlobSlice::~BlobSlice() {}
+
+BlobStorageContext::BlobStorageContext()
+    : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()),
+      ptr_factory_(this) {}
 
 BlobStorageContext::~BlobStorageContext() {
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID(
     const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  if (!entry)
     return nullptr;
-  }
-  return base::WrapUnique(
-      new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
-                         this, base::ThreadTaskRunnerHandle::Get().get()));
+  return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL(
     const GURL& url) {
   std::string uuid;
-  BlobRegistryEntry* entry = registry_.GetEntryFromURL(url, &uuid);
-  if (!entry) {
+  BlobEntry* entry = registry_.GetEntryFromURL(url, &uuid);
+  if (!entry)
     return nullptr;
-  }
-  return base::WrapUnique(
-      new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
-                         this, base::ThreadTaskRunnerHandle::Get().get()));
+  return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
     const BlobDataBuilder& external_builder) {
   TRACE_EVENT0("Blob", "Context::AddFinishedBlob");
-  CreatePendingBlob(external_builder.uuid(), external_builder.content_type_,
-                    external_builder.content_disposition_);
-  CompletePendingBlob(external_builder);
-  std::unique_ptr<BlobDataHandle> handle =
-      GetBlobDataFromUUID(external_builder.uuid_);
-  DecrementBlobRefCount(external_builder.uuid_);
-  return handle;
+  return BuildBlob(external_builder, TransportAllowedCallback());
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
@@ -78,123 +305,167 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
   return AddFinishedBlob(*builder);
 }
 
+std::unique_ptr<BlobDataHandle> BlobStorageContext::AddBrokenBlob(
+    const std::string& uuid,
+    const std::string& content_type,
+    const std::string& content_disposition,
+    BlobStatus reason) {
+  DCHECK(!registry_.HasEntry(uuid));
+  DCHECK(BlobStatusIsError(reason));
+  BlobEntry* entry =
+      registry_.CreateEntry(uuid, content_type, content_disposition);
+  entry->status_ = reason;
+  FinishBuilding(entry);
+  return CreateHandle(uuid, entry);
+}
+
 bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url,
                                                const std::string& uuid) {
-  if (!registry_.CreateUrlMapping(blob_url, uuid)) {
+  if (!registry_.CreateUrlMapping(blob_url, uuid))
     return false;
-  }
   IncrementBlobRefCount(uuid);
   return true;
 }
 
 void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) {
   std::string uuid;
-  if (!registry_.DeleteURLMapping(blob_url, &uuid)) {
+  if (!registry_.DeleteURLMapping(blob_url, &uuid))
     return;
-  }
   DecrementBlobRefCount(uuid);
 }
 
-void BlobStorageContext::CreatePendingBlob(
-    const std::string& uuid,
-    const std::string& content_type,
-    const std::string& content_disposition) {
-  DCHECK(!registry_.GetEntry(uuid) && !uuid.empty());
-  registry_.CreateEntry(uuid, content_type, content_disposition);
-}
+std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
+    const BlobDataBuilder& content,
+    const TransportAllowedCallback& can_populate_memory) {
+  DCHECK(!registry_.HasEntry(content.uuid_));
 
-void BlobStorageContext::CompletePendingBlob(
-    const BlobDataBuilder& external_builder) {
-  BlobRegistryEntry* entry = registry_.GetEntry(external_builder.uuid());
-  DCHECK(entry);
-  DCHECK(!entry->data.get()) << "Blob already constructed: "
-                             << external_builder.uuid();
-  // We want to handle storing our broken blob as well.
-  switch (entry->state) {
-    case BlobState::PENDING: {
-      entry->data_builder.reset(new InternalBlobData::Builder());
-      InternalBlobData::Builder* internal_data_builder =
-          entry->data_builder.get();
-
-      bool broken = false;
-      for (const auto& blob_item : external_builder.items_) {
-        IPCBlobCreationCancelCode error_code;
-        if (!AppendAllocatedBlobItem(external_builder.uuid_, blob_item,
-                                     internal_data_builder, &error_code)) {
-          broken = true;
-          memory_usage_ -= entry->data_builder->GetNonsharedMemoryUsage();
-          entry->state = BlobState::BROKEN;
-          entry->broken_reason = error_code;
-          entry->data_builder.reset(new InternalBlobData::Builder());
-          break;
-        }
-      }
-      entry->data = entry->data_builder->Build();
-      entry->data_builder.reset();
-      entry->state = broken ? BlobState::BROKEN : BlobState::COMPLETE;
-      break;
-    }
-    case BlobState::BROKEN: {
-      InternalBlobData::Builder builder;
-      entry->data = builder.Build();
-      break;
+  BlobEntry* entry = registry_.CreateEntry(content.uuid_, content.content_type_,
+                                           content.content_disposition_);
+
+  // This flattens all blob references in the transportion content out and
+  // stores the complete item representation in the internal data.
+  BlobFlattener flattener(content, entry, &registry_);
+
+  // If we contain invalid references we're invalid input.
+  if (flattener.status == BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS) {
+    CancelBuildingBlob(content.uuid_,
+                       BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  // If we contain broken references (or we reference ourself) we want to fail.
+  if (flattener.status == BlobStatus::ERR_REFERENCED_BLOB_BROKEN) {
+    CancelBuildingBlob(content.uuid_, BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) {
+    CancelBuildingBlob(content.uuid_, BlobStatus::ERR_OUT_OF_MEMORY);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  entry->status_ = flattener.status;
+  entry->size_ = flattener.total_size;
+  std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry);
+
+  size_t num_building_dependent_blobs = 0;
+  std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs;
+  // We hold a handle to all blobs we're using. This is important, as our memory
+  // accounting can be delayed until OnEnoughSizeForBlobData is called, and we
+  // only free memory on canceling when we've done this accounting. If a
+  // dependent blob is dereferenced, then we're the last blob holding onto that
+  // data item, and we need to account for that. So we prevent that case by
+  // holding onto all blobs.
+  for (const std::pair<std::string, BlobEntry*>& pending_blob :
+       flattener.dependent_blobs) {
+    dependent_blobs.push_back(
+        CreateHandle(pending_blob.first, pending_blob.second));
+    if (BlobStatusIsPending(pending_blob.second->status())) {
+      pending_blob.second->building_state_->build_completion_callbacks
+          .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished,
+                                ptr_factory_.GetWeakPtr(), content.uuid_));
+      num_building_dependent_blobs++;
     }
-    case BlobState::COMPLETE:
-      DCHECK(false) << "Blob already constructed: " << external_builder.uuid();
-      return;
   }
 
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->data->items().size());
-  UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken",
-                        entry->state == BlobState::BROKEN);
-  if (entry->state == BlobState::BROKEN) {
-    UMA_HISTOGRAM_ENUMERATION(
-        "Storage.Blob.BrokenReason", static_cast<int>(entry->broken_reason),
-        (static_cast<int>(IPCBlobCreationCancelCode::LAST) + 1));
+  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+         !can_populate_memory)
+      << "There is no pending content for the user to populate, so the "
+         "callback should be null.";
+  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+         can_populate_memory)
+      << "If we have pending content then there needs to be a callback "
+         "present.";
+
+  entry->building_state_.reset(new BlobEntry::BuildingState(
+      !flattener.user_items.empty(), can_populate_memory,
+      num_building_dependent_blobs, flattener.memory_quota_needed > 0));
+
+  BlobEntry::BuildingState* building_state = entry->building_state_.get();
+  std::swap(building_state->copies, flattener.copies);
+  std::swap(building_state->dependent_blobs, dependent_blobs);
+  std::swap(building_state->user_items, flattener.user_items);
+
+  if (flattener.memory_quota_needed > 0) {
+    // We can finish our blob before returning from this method.
+    base::WeakPtr<QuotaAllocationTask> pending_request =
+        memory_controller_.ReserveMemoryQuota(
+            std::move(flattener.pending_memory_items),
+            base::Bind(&BlobStorageContext::OnEnoughSizeForMemory,
+                       ptr_factory_.GetWeakPtr(), content.uuid_));
+    if (entry->building_state_)

[kinuko, 2016/11/10 05:16:36]

When and why this could be null?

[dmurph, 2016/11/10 19:53:19]

We can finish our blob before that method returns (see the above comment). Then
our building state will be null. I'll update the comment.

+      entry->building_state_->memory_quota_request = std::move(pending_request);
   }
-  size_t total_memory = 0, nonshared_memory = 0;
-  entry->data->GetMemoryUsage(&total_memory, &nonshared_memory);
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", total_memory / 1024);
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
-                       nonshared_memory / 1024);
-  TRACE_COUNTER1("Blob", "MemoryStoreUsageBytes", memory_usage_);
 
-  auto runner = base::ThreadTaskRunnerHandle::Get();
-  for (const auto& callback : entry->build_completion_callbacks) {
-    runner->PostTask(FROM_HERE,
-                     base::Bind(callback, entry->state == BlobState::COMPLETE,
-                                entry->broken_reason));
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+
+  return handle;
+}
+
+void BlobStorageContext::CancelBuildingBlob(const std::string& uuid,
+                                            BlobStatus reason) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  DCHECK(entry);
+  DCHECK(BlobStatusIsError(reason));
+  TransportAllowedCallback user_data_population_callback;
+  if (entry->building_state_ &&
+      entry->building_state_->transport_allowed_callback) {
+    user_data_population_callback =
+        entry->building_state_->transport_allowed_callback;
+    entry->building_state_->transport_allowed_callback.Reset();
   }
-  entry->build_completion_callbacks.clear();
+  ClearAndFreeMemory(uuid, entry);
+  entry->status_ = reason;
+  if (user_data_population_callback) {
+    user_data_population_callback.Run(
+        reason, std::vector<BlobMemoryController::FileCreationInfo>());
+  }
+  entry->items_.clear();
+  entry->offsets_.clear();
+  entry->size_ = 0;
+  FinishBuilding(entry);
 }
 
-void BlobStorageContext::CancelPendingBlob(const std::string& uuid,
-                                           IPCBlobCreationCancelCode reason) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  DCHECK(entry && entry->state == BlobState::PENDING);
-  entry->state = BlobState::BROKEN;
-  entry->broken_reason = reason;
-  CompletePendingBlob(BlobDataBuilder(uuid));
+void BlobStorageContext::NotifyTransportComplete(const std::string& uuid) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  CHECK(entry) << "There is no blob entry with uuid " << uuid;
+  DCHECK(BlobStatusIsPending(entry->status()));
+  NotifyTransportComplete(entry);
 }
 
 void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+  BlobEntry* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  ++(entry->refcount);
+  ++(entry->refcount_);
 }
 
 void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+  BlobEntry* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  DCHECK_GT(entry->refcount, 0u);
-  if (--(entry->refcount) == 0) {
-    size_t memory_freeing = 0;
-    if (entry->state == BlobState::COMPLETE) {
-      memory_freeing = entry->data->GetUnsharedMemoryUsage();
-      entry->data->RemoveBlobFromShareableItems(uuid);
-    }
-    DCHECK_LE(memory_freeing, memory_usage_);
-    memory_usage_ -= memory_freeing;
+  DCHECK_GT(entry->refcount_, 0u);
+  if (--(entry->refcount_) == 0) {
+    ClearAndFreeMemory(uuid, entry);
     registry_.DeleteEntry(uuid);
   }
 }
@@ -202,263 +473,185 @@ void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
 std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot(
     const std::string& uuid) {
   std::unique_ptr<BlobDataSnapshot> result;
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (entry->state != BlobState::COMPLETE) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  if (entry->status() != BlobStatus::DONE)
     return result;
-  }
 
-  const InternalBlobData& data = *entry->data;
   std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot(
-      uuid, entry->content_type, entry->content_disposition));
-  snapshot->items_.reserve(data.items().size());
-  for (const auto& shareable_item : data.items()) {
+      uuid, entry->content_type(), entry->content_disposition()));
+  snapshot->items_.reserve(entry->items().size());
+  for (const auto& shareable_item : entry->items()) {
     snapshot->items_.push_back(shareable_item->item());
   }
+  memory_controller_.NotifyMemoryItemsUsed(entry->items());
   return snapshot;
 }
 
-bool BlobStorageContext::IsBroken(const std::string& uuid) const {
-  const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
-    return true;
-  }
-  return entry->state == BlobState::BROKEN;
+BlobStatus BlobStorageContext::GetBlobStatus(const std::string& uuid) const {
+  const BlobEntry* entry = registry_.GetEntry(uuid);
+  if (!entry)
+    return BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+  return entry->status();
 }
 
-bool BlobStorageContext::IsBeingBuilt(const std::string& uuid) const {
-  const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
-    return false;
+void BlobStorageContext::RunOnConstructionComplete(
+    const std::string& uuid,
+    const BlobStatusCallback& done) {
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  DCHECK(entry);
+  if (BlobStatusIsPending(entry->status())) {
+    entry->building_state_->build_completion_callbacks.push_back(done);
+    return;
   }
-  return entry->state == BlobState::PENDING;
+  done.Run(entry->status());
 }
 
-void BlobStorageContext::RunOnConstructionComplete(
+std::unique_ptr<BlobDataHandle> BlobStorageContext::CreateHandle(
     const std::string& uuid,
-    const BlobConstructedCallback& done) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+    BlobEntry* entry) {
+  return base::WrapUnique(new BlobDataHandle(
+      uuid, entry->content_type_, entry->content_disposition_, entry->size_,
+      this, base::ThreadTaskRunnerHandle::Get().get()));
+}
+
+void BlobStorageContext::NotifyTransportComplete(BlobEntry* entry) {
   DCHECK(entry);
-  switch (entry->state) {
-    case BlobState::COMPLETE:
-      done.Run(true, IPCBlobCreationCancelCode::UNKNOWN);
-      return;
-    case BlobState::BROKEN:
-      done.Run(false, entry->broken_reason);
-      return;
-    case BlobState::PENDING:
-      entry->build_completion_callbacks.push_back(done);
-      return;
-  }
-  NOTREACHED();
+  SetItemStateToPopulated(&entry->building_state_->user_items);
+  entry->status_ = BlobStatus::PENDING_INTERNALS;
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
 }
 
-bool BlobStorageContext::AppendAllocatedBlobItem(
-    const std::string& target_blob_uuid,
-    scoped_refptr<BlobDataItem> blob_item,
-    InternalBlobData::Builder* target_blob_builder,
-    IPCBlobCreationCancelCode* error_code) {
-  DCHECK(error_code);
-  *error_code = IPCBlobCreationCancelCode::UNKNOWN;
-  bool error = false;
-
-  // The blob data is stored in the canonical way which only contains a
-  // list of Data, File, and FileSystem items. Aggregated TYPE_BLOB items
-  // are expanded into the primitive constituent types and reused if possible.
-  // 1) The Data item is denoted by the raw data and length.
-  // 2) The File item is denoted by the file path, the range and the expected
-  //    modification time.
-  // 3) The FileSystem File item is denoted by the FileSystem URL, the range
-  //    and the expected modification time.
-  // 4) The Blob item is denoted by the source blob and an offset and size.
-  //    Internal items that are fully used by the new blob (not cut by the
-  //    offset or size) are shared between the blobs.  Otherwise, the relevant
-  //    portion of the item is copied.
-
-  DCHECK(blob_item->data_element_ptr());
-  const DataElement& data_element = blob_item->data_element();
-  uint64_t length = data_element.length();
-  uint64_t offset = data_element.offset();
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
-                       memory_usage_ / 1024);
-  switch (data_element.type()) {
-    case DataElement::TYPE_BYTES:
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024);
-      DCHECK(!offset);
-      if (memory_usage_ + length > kBlobStorageMaxMemoryUsage) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
-        break;
-      }
-      memory_usage_ += length;
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITH_QUOTA));
-      break;
-    case DataElement::TYPE_FILE: {
-      bool full_file = (length == std::numeric_limits<uint64_t>::max());
-      UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file);
-      if (!full_file) {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File",
-                             (length - offset) / 1024);
-      }
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_FILE_FILESYSTEM: {
-      bool full_file = (length == std::numeric_limits<uint64_t>::max());
-      UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown",
-                            full_file);
-      if (!full_file) {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem",
-                             (length - offset) / 1024);
-      }
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_BLOB: {
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob",
-                           (length - offset) / 1024);
-      // We grab the handle to ensure it stays around while we copy it.
-      std::unique_ptr<BlobDataHandle> src =
-          GetBlobDataFromUUID(data_element.blob_uuid());
-      if (!src || src->IsBroken() || src->IsBeingBuilt()) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN;
-        break;
-      }
-      BlobRegistryEntry* other_entry =
-          registry_.GetEntry(data_element.blob_uuid());
-      DCHECK(other_entry->data);
-      if (!AppendBlob(target_blob_uuid, *other_entry->data, offset, length,
-                      target_blob_builder)) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
+void BlobStorageContext::FinishBuilding(BlobEntry* entry) {
+  DCHECK(entry);
+
+  if (BlobStatusIsPending(entry->status_)) {

[kinuko, 2016/11/10 05:16:36]

Is PENDING_INTERNALS used mostly for this if block?  Setting entry state to
PENDING when we're done feels a bit weird.

[dmurph, 2016/11/10 19:53:19]

Yes. We're still not done if we haven't copied over all of the dependent blob
items.

PENDING_INTERNALS is also used for us waiting for dependent blobs. Those
status's don't have a super great 1:1 mapping of our internals, as the dependent
blob stuff can be happening during quota request & transportation. The
constraints are:
1. We need quota before transportation
2. We need quota before copying memory from dependent blobs.

When we introduce disk, it gets more complicated as we CAN be requesting one
quota type (file) for transportation and another (memory) for copies. So a
linear state system doesn't really work later. But it mostly works here.

+    for (const ItemCopyEntry& copy : entry->building_state_->copies) {
+      // Our source item can be a file if it was a slice of an unpopulated file,
+      // or a slice of data that was then paged to disk.
+      size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length());
+      DataElement::Type dest_type = copy.dest_item->item()->type();
+      switch (copy.source_item->item()->type()) {
+        case DataElement::TYPE_BYTES: {
+          DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION);
+          const char* src_data =
+              copy.source_item->item()->bytes() + copy.source_item_offset;
+          copy.dest_item->item()->item_->SetToBytes(src_data, dest_size);
+        } break;
+        case DataElement::TYPE_FILE:
+        case DataElement::TYPE_UNKNOWN:
+        case DataElement::TYPE_BLOB:
+        case DataElement::TYPE_BYTES_DESCRIPTION:
+        case DataElement::TYPE_FILE_FILESYSTEM:
+        case DataElement::TYPE_DISK_CACHE_ENTRY:
+          NOTREACHED();
+          break;
       }
-      break;
+      copy.dest_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
     }
-    case DataElement::TYPE_DISK_CACHE_ENTRY: {
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry",
-                           (length - offset) / 1024);
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_BYTES_DESCRIPTION:
-    case DataElement::TYPE_UNKNOWN:
-      NOTREACHED();
-      break;
+    entry->status_ = BlobStatus::DONE;
+  }
+
+  std::vector<BlobStatusCallback> callbacks;
+  if (entry->building_state_.get()) {
+    std::swap(callbacks, entry->building_state_->build_completion_callbacks);
+    entry->building_state_.reset();
+  }
+
+  memory_controller_.NotifyMemoryItemsUsed(entry->items());
+
+  auto runner = base::ThreadTaskRunnerHandle::Get();
+  for (const auto& callback : callbacks)
+    runner->PostTask(FROM_HERE, base::Bind(callback, entry->status()));
+
+  for (const auto& shareable_item : entry->items()) {
+    DCHECK_NE(DataElement::TYPE_BYTES_DESCRIPTION,
+              shareable_item->item()->type());
+    DCHECK(shareable_item->IsPopulated()) << shareable_item->state();
   }
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend",
-                       memory_usage_ / 1024);
-  return !error;
 }
 
-bool BlobStorageContext::AppendBlob(
-    const std::string& target_blob_uuid,
-    const InternalBlobData& blob,
-    uint64_t offset,
-    uint64_t length,
-    InternalBlobData::Builder* target_blob_builder) {
-  DCHECK_GT(length, 0ull);
-
-  const std::vector<scoped_refptr<ShareableBlobDataItem>>& items = blob.items();
-  auto iter = items.begin();
-  if (offset) {
-    for (; iter != items.end(); ++iter) {
-      const BlobDataItem& item = *(iter->get()->item());
-      if (offset >= item.length())
-        offset -= item.length();
-      else
-        break;
-    }
+void BlobStorageContext::RequestTransport(
+    BlobEntry* entry,
+    std::vector<BlobMemoryController::FileCreationInfo> files) {
+  BlobEntry::BuildingState* building_state = entry->building_state_.get();
+  if (building_state->transport_allowed_callback) {
+    TransportAllowedCallback user_data_population_callback =
+        building_state->transport_allowed_callback;
+    building_state->transport_allowed_callback.Reset();
+    user_data_population_callback.Run(BlobStatus::PENDING_TRANSPORT,
+                                      std::move(files));
+    return;
   }
+  DCHECK(files.empty());
+  NotifyTransportComplete(entry);
+}
 
-  for (; iter != items.end() && length > 0; ++iter) {
-    scoped_refptr<ShareableBlobDataItem> shareable_item = iter->get();
-    const BlobDataItem& item = *(shareable_item->item());
-    uint64_t item_length = item.length();
-    DCHECK_GT(item_length, offset);
-    uint64_t current_length = item_length - offset;
-    uint64_t new_length = current_length > length ? length : current_length;
-
-    bool reusing_blob_item = offset == 0 && new_length == item.length();
-    UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item);
-    if (reusing_blob_item) {
-      shareable_item->referencing_blobs_mutable()->insert(target_blob_uuid);
-      target_blob_builder->AppendSharedBlobItem(shareable_item);
-      length -= new_length;
-      continue;
-    }
+void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid,
+                                               bool success) {
+  if (!success) {
+    CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+    return;
+  }
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  if (!entry || !entry->building_state_.get())
+    return;
+  BlobEntry::BuildingState& building_state = *entry->building_state_;
+  DCHECK(!building_state.memory_quota_request);
+
+  if (building_state.transport_items_present) {
+    entry->status_ = BlobStatus::PENDING_TRANSPORT;
+    RequestTransport(entry,
+                     std::vector<BlobMemoryController::FileCreationInfo>());
+  } else {
+    entry->status_ = BlobStatus::PENDING_INTERNALS;
+  }
 
-    // We need to do copying of the items when we have a different offset or
-    // length
-    switch (item.type()) {
-      case DataElement::TYPE_BYTES: {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes",
-                             new_length / 1024);
-        if (memory_usage_ + new_length > kBlobStorageMaxMemoryUsage) {
-          return false;
-        }
-        DCHECK(!item.offset());
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToBytes(item.bytes() + offset,
-                            static_cast<int64_t>(new_length));
-        memory_usage_ += new_length;
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid, new BlobDataItem(std::move(element)),
-            ShareableBlobDataItem::POPULATED_WITH_QUOTA));
-      } break;
-      case DataElement::TYPE_FILE: {
-        DCHECK_NE(item.length(), std::numeric_limits<uint64_t>::max())
-            << "We cannot use a section of a file with an unknown length";
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File",
-                             new_length / 1024);
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToFilePathRange(item.path(), item.offset() + offset,
-                                    new_length,
-                                    item.expected_modification_time());
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid,
-            new BlobDataItem(std::move(element), item.data_handle_),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_FILE_FILESYSTEM: {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
-                             new_length / 1024);
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToFileSystemUrlRange(item.filesystem_url(),
-                                         item.offset() + offset, new_length,
-                                         item.expected_modification_time());
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid, new BlobDataItem(std::move(element)),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_DISK_CACHE_ENTRY: {
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToDiskCacheEntryRange(item.offset() + offset,
-                                          new_length);
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid,
-            new BlobDataItem(std::move(element), item.data_handle_,
-                             item.disk_cache_entry(),
-                             item.disk_cache_stream_index(),
-                             item.disk_cache_side_stream_index()),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_BYTES_DESCRIPTION:
-      case DataElement::TYPE_BLOB:
-      case DataElement::TYPE_UNKNOWN:
-        CHECK(false) << "Illegal blob item type: " << item.type();
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::OnDependentBlobFinished(
+    const std::string& owning_blob_uuid,
+    BlobStatus status) {
+  BlobEntry* entry = registry_.GetEntry(owning_blob_uuid);
+  if (!entry || !entry->building_state_)
+    return;
+
+  if (BlobStatusIsError(status)) {
+    DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status)
+        << "Referenced blob should never be dereferenced while we "
+        << "are depending on it, as our system holds a handle.";
+    CancelBuildingBlob(owning_blob_uuid,
+                       BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+    return;
+  }
+  DCHECK_GT(entry->building_state_->num_building_dependent_blobs, 0u);
+  --entry->building_state_->num_building_dependent_blobs;
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::ClearAndFreeMemory(const std::string& uuid,
+                                            BlobEntry* entry) {
+  if (entry->building_state_) {
+    BlobEntry::BuildingState* building_state = entry->building_state_.get();
+    if (building_state->memory_quota_request) {
+      building_state->memory_quota_request->Cancel();
     }
-    length -= new_length;
-    offset = 0;
   }
-  return true;
+  entry->RemoveBlobFromShareableItems(uuid);
+  entry->items_.clear();
+  entry->offsets_.clear();
+}
+
+void BlobStorageContext::SetItemStateToPopulated(
+    std::vector<ShareableBlobDataItem*>* items) {
+  for (ShareableBlobDataItem* shareable_item : *items) {

[kinuko, 2016/11/10 05:16:36]

nit: I'd probably just put this 4-lines loop directly in NotifyTransportComplete

[dmurph, 2016/11/10 19:53:19]

Done.

+    DCHECK(shareable_item->state() == ShareableBlobDataItem::QUOTA_GRANTED);
+    shareable_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
+  }
 }
 
 }  // namespace storage