[BlobAsync] Moving async handling into BlobStorageContext & quota out.

storage/browser/blob/blob_storage_context.cc

Index: storage/browser/blob/blob_storage_context.cc
diff --git a/storage/browser/blob/blob_storage_context.cc b/storage/browser/blob/blob_storage_context.cc
index ec66525e23f1746afa84a746f13b14ef5dc3b11b..a5ff5e198ee620ac9ed570775218b14f70f624a0 100644
--- a/storage/browser/blob/blob_storage_context.cc
+++ b/storage/browser/blob/blob_storage_context.cc
@@ -4,12 +4,10 @@
 
 #include "storage/browser/blob/blob_storage_context.h"
 
-#include <stddef.h>
-#include <stdint.h>
-
 #include <algorithm>
 #include <limits>
 #include <memory>
+#include <set>
 #include <utility>
 
 #include "base/bind.h"
@@ -19,57 +17,288 @@
 #include "base/memory/ptr_util.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "base/task_runner.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/trace_event/trace_event.h"
+#include "storage/browser/blob/blob_async_transport_request_builder.h"
 #include "storage/browser/blob/blob_data_builder.h"
-#include "storage/browser/blob/blob_data_handle.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/blob_data_snapshot.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
 #include "url/gurl.h"
 
 namespace storage {
-using BlobRegistryEntry = BlobStorageRegistry::Entry;
-using BlobState = BlobStorageRegistry::BlobState;
+namespace {
+using ItemCopyEntry = InternalBlobData::ItemCopyEntry;
+using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+
+bool IsBytes(DataElement::Type type) {
+  return type == DataElement::TYPE_BYTES ||
+         type == DataElement::TYPE_BYTES_DESCRIPTION;
+}
+}  // namespace
+
+BlobStorageContext::BlobFlattener::BlobFlattener(
+    const BlobDataBuilder& input_builder,
+    InternalBlobData* output_blob,
+    BlobStorageRegistry* registry) {
+  const std::string& uuid = input_builder.uuid_;
+  std::set<std::string> dependent_blob_uuids;
+
+  size_t num_files_with_unknown_size = 0;
+  size_t num_building_dependent_blobs = 0;
+
+  base::CheckedNumeric<uint64_t> checked_total_size = 0;
+  base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0;
+
+  for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) {
+    const DataElement& input_element = input_item->data_element();
+    DataElement::Type type = input_element.type();
+    uint64_t length = input_element.length();
+
+    if (IsBytes(type)) {
+      DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length());
+      checked_memory_quota_needed += length;
+      checked_total_size += length;
+      scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+          std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED);
+      pending_memory_items.push_back(item);
+      user_items.push_back(item.get());
+      output_blob->AppendSharedBlobItem(uuid, std::move(item));
+      continue;
+    }
+    if (type == DataElement::TYPE_BLOB) {
+      InternalBlobData* ref_entry =
+          registry->GetEntry(input_element.blob_uuid());
+
+      if (!ref_entry || input_element.blob_uuid() == uuid) {
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      if (BlobStatusIsError(ref_entry->status())) {
+        status = BlobStatus::ERR_REFERENCED_BLOB_BROKEN;
+        return;
+      }
+
+      if (ref_entry->total_size() == DataElement::kUnknownSize) {
+        // We can't reference a blob with unknown size.
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      if (dependent_blob_uuids.find(input_element.blob_uuid()) ==
+          dependent_blob_uuids.end()) {
+        dependent_blobs.push_back(
+            std::make_pair(input_element.blob_uuid(), ref_entry));
+        dependent_blob_uuids.insert(input_element.blob_uuid());
+        if (BlobStatusIsPending(ref_entry->status())) {
+          num_building_dependent_blobs++;
+        }
+      }
+
+      length = length == DataElement::kUnknownSize ? ref_entry->total_size()
+                                                   : input_element.length();
+      checked_total_size += length;
+
+      // If we're referencing the whole blob, then we don't need to slice.
+      if (input_element.offset() == 0 && length == ref_entry->total_size()) {
+        for (const auto& shareable_item : ref_entry->items()) {
+          output_blob->AppendSharedBlobItem(uuid, shareable_item);
+        }
+        continue;
+      }
+
+      // Validate our reference has good offset & length.
+      if (input_element.offset() + length > ref_entry->total_size()) {
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+
+      BlobSlice slice(*ref_entry, input_element.offset(), length);
+
+      if (slice.first_source_item) {
+        copies.push_back(ItemCopyEntry(slice.first_source_item,
+                                       slice.first_item_slice_offset,
+                                       slice.dest_items.front()));
+        pending_memory_items.push_back(slice.dest_items.front());
+      }
+      if (slice.last_source_item) {
+        copies.push_back(
+            ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back()));
+        pending_memory_items.push_back(slice.dest_items.back());
+      }
+      checked_memory_quota_needed += slice.copying_memory_size;
+
+      for (auto& shareable_item : slice.dest_items) {
+        output_blob->AppendSharedBlobItem(uuid, std::move(shareable_item));
+      }
+      continue;
+    }
+
+    DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element))
+        << "File allocation not implemented.";
+    if (length == DataElement::kUnknownSize)
+      num_files_with_unknown_size++;
+
+    scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+        std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
+
+    checked_total_size += length;
+    output_blob->AppendSharedBlobItem(uuid, std::move(item));
+  }
+
+  if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) {
+    status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+    return;
+  }
+  if (!checked_total_size.IsValid() || !checked_memory_quota_needed.IsValid()) {
+    status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+    return;
+  }
+  total_size = checked_total_size.ValueOrDie();
+  memory_quota_needed = checked_memory_quota_needed.ValueOrDie();
+  if (memory_quota_needed) {
+    status = BlobStatus::PENDING_QUOTA;
+  } else {
+    status = BlobStatus::PENDING_INTERNALS;
+  }
+}
+
+BlobStorageContext::BlobFlattener::~BlobFlattener() {}
+
+BlobStorageContext::BlobSlice::BlobSlice(const InternalBlobData& source,
+                                         uint64_t slice_offset,
+                                         uint64_t slice_size) {
+  const auto& source_items = source.items();
+  const auto& offsets = source.offsets();
+  DCHECK_LE(slice_offset + slice_size, source.total_size());
+  size_t item_index =
+      std::upper_bound(offsets.begin(), offsets.end(), slice_offset) -
+      offsets.begin();
+  uint64_t item_offset =
+      item_index == 0 ? slice_offset : slice_offset - offsets[item_index - 1];
+  size_t num_items = source_items.size();
+
+  size_t first_item_index = item_index;
+  copying_memory_size = 0;
+
+  // Read starting from 'first_item_index' and 'item_offset'.
+  for (uint64_t total_sliced = 0;
+       item_index < num_items && total_sliced < slice_size; item_index++) {
+    const scoped_refptr<BlobDataItem>& source_item =
+        source_items[item_index]->item();
+    uint64_t source_length = source_item->length();
+    DCHECK_NE(source_length, std::numeric_limits<uint64_t>::max());
+    DCHECK_NE(source_length, 0ull);
+
+    uint64_t read_size =
+        std::min(source_length - item_offset, slice_size - total_sliced);
+    total_sliced += read_size;
+
+    if (read_size == source_length) {
+      // We can share the entire item.
+      dest_items.push_back(source_items[item_index]);
+      continue;
+    }
 
-BlobStorageContext::BlobStorageContext() : memory_usage_(0) {}
+    scoped_refptr<BlobDataItem> data_item;
+    ShareableBlobDataItem::State state =
+        ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA;
+    switch (source_item->type()) {
+      case DataElement::TYPE_BYTES_DESCRIPTION:
+      case DataElement::TYPE_BYTES: {
+        if (item_index == first_item_index) {
+          first_item_slice_offset = item_offset;
+          first_source_item = source_items[item_index];
+        } else {
+          last_source_item = source_items[item_index];
+        }
+        copying_memory_size += read_size;
+        // Since we don't have quota yet for memory, we create temporary items
+        // for this data. When our blob is finished constructing, all dependent
+        // blobs are done, and we have enough memory quota, we'll copy the data
+        // over.
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToBytesDescription(base::checked_cast<size_t>(read_size));
+        data_item = new BlobDataItem(std::move(element));
+        state = ShareableBlobDataItem::QUOTA_NEEDED;
+        break;
+      }
+      case DataElement::TYPE_FILE: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToFilePathRange(
+            source_item->path(), source_item->offset() + item_offset, read_size,
+            source_item->expected_modification_time());
+        data_item =
+            new BlobDataItem(std::move(element), source_item->data_handle_);
+
+        DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element()))
+            << "File allocation unimplemented.";
+        break;
+      }
+      case DataElement::TYPE_FILE_FILESYSTEM: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToFileSystemUrlRange(
+            source_item->filesystem_url(), source_item->offset() + item_offset,
+            read_size, source_item->expected_modification_time());
+        data_item = new BlobDataItem(std::move(element));
+        break;
+      }
+      case DataElement::TYPE_DISK_CACHE_ENTRY: {
+        std::unique_ptr<DataElement> element(new DataElement());
+        element->SetToDiskCacheEntryRange(source_item->offset() + item_offset,
+                                          read_size);
+        data_item =
+            new BlobDataItem(std::move(element), source_item->data_handle_,
+                             source_item->disk_cache_entry(),
+                             source_item->disk_cache_stream_index(),
+                             source_item->disk_cache_side_stream_index());
+        break;
+      }
+      case DataElement::TYPE_BLOB:
+      case DataElement::TYPE_UNKNOWN:
+        CHECK(false) << "Illegal blob item type: " << source_item->type();
+    }
+    dest_items.push_back(
+        new ShareableBlobDataItem(std::move(data_item), state));
+    item_offset = 0;
+  }
+}
+
+BlobStorageContext::BlobSlice::~BlobSlice() {}
+
+BlobStorageContext::BlobStorageContext()
+    : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()),
+      ptr_factory_(this) {}
 
 BlobStorageContext::~BlobStorageContext() {
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID(
     const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (!entry)
     return nullptr;
-  }
-  return base::WrapUnique(
-      new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
-                         this, base::ThreadTaskRunnerHandle::Get().get()));
+  return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL(
     const GURL& url) {
   std::string uuid;
-  BlobRegistryEntry* entry = registry_.GetEntryFromURL(url, &uuid);
-  if (!entry) {
+  InternalBlobData* entry = registry_.GetEntryFromURL(url, &uuid);
+  if (!entry)
     return nullptr;
-  }
-  return base::WrapUnique(
-      new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
-                         this, base::ThreadTaskRunnerHandle::Get().get()));
+  return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
     const BlobDataBuilder& external_builder) {
   TRACE_EVENT0("Blob", "Context::AddFinishedBlob");
-  CreatePendingBlob(external_builder.uuid(), external_builder.content_type_,
-                    external_builder.content_disposition_);
-  CompletePendingBlob(external_builder);
-  std::unique_ptr<BlobDataHandle> handle =
-      GetBlobDataFromUUID(external_builder.uuid_);
-  DecrementBlobRefCount(external_builder.uuid_);
-  return handle;
+  return BuildBlob(external_builder, PopulatationAllowedCallback());
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
@@ -78,123 +307,172 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
   return AddFinishedBlob(*builder);
 }
 
+std::unique_ptr<BlobDataHandle> BlobStorageContext::AddBrokenBlob(
+    const std::string& uuid,
+    const std::string& content_type,
+    const std::string& content_disposition,
+    BlobStatus reason) {
+  DCHECK(!registry_.HasEntry(uuid));
+  DCHECK(BlobStatusIsError(reason));
+  InternalBlobData* entry =
+      registry_.CreateEntry(uuid, content_type, content_disposition);
+  entry->status_ = reason;
+  FinishBuilding(entry);
+  return CreateHandle(uuid, entry);
+}
+
 bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url,
                                                const std::string& uuid) {
-  if (!registry_.CreateUrlMapping(blob_url, uuid)) {
+  if (!registry_.CreateUrlMapping(blob_url, uuid))
     return false;
-  }
   IncrementBlobRefCount(uuid);
   return true;
 }
 
 void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) {
   std::string uuid;
-  if (!registry_.DeleteURLMapping(blob_url, &uuid)) {
+  if (!registry_.DeleteURLMapping(blob_url, &uuid))
     return;
-  }
   DecrementBlobRefCount(uuid);
 }
 
-void BlobStorageContext::CreatePendingBlob(
-    const std::string& uuid,
-    const std::string& content_type,
-    const std::string& content_disposition) {
-  DCHECK(!registry_.GetEntry(uuid) && !uuid.empty());
-  registry_.CreateEntry(uuid, content_type, content_disposition);
-}
+std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
+    const BlobDataBuilder& content,
+    const PopulatationAllowedCallback& can_populate_memory) {
+  DCHECK(!registry_.HasEntry(content.uuid_));
 
-void BlobStorageContext::CompletePendingBlob(
-    const BlobDataBuilder& external_builder) {
-  BlobRegistryEntry* entry = registry_.GetEntry(external_builder.uuid());
-  DCHECK(entry);
-  DCHECK(!entry->data.get()) << "Blob already constructed: "
-                             << external_builder.uuid();
-  // We want to handle storing our broken blob as well.
-  switch (entry->state) {
-    case BlobState::PENDING: {
-      entry->data_builder.reset(new InternalBlobData::Builder());
-      InternalBlobData::Builder* internal_data_builder =
-          entry->data_builder.get();
-
-      bool broken = false;
-      for (const auto& blob_item : external_builder.items_) {
-        IPCBlobCreationCancelCode error_code;
-        if (!AppendAllocatedBlobItem(external_builder.uuid_, blob_item,
-                                     internal_data_builder, &error_code)) {
-          broken = true;
-          memory_usage_ -= entry->data_builder->GetNonsharedMemoryUsage();
-          entry->state = BlobState::BROKEN;
-          entry->broken_reason = error_code;
-          entry->data_builder.reset(new InternalBlobData::Builder());
-          break;
-        }
-      }
-      entry->data = entry->data_builder->Build();
-      entry->data_builder.reset();
-      entry->state = broken ? BlobState::BROKEN : BlobState::COMPLETE;
-      break;
-    }
-    case BlobState::BROKEN: {
-      InternalBlobData::Builder builder;
-      entry->data = builder.Build();
-      break;
+  InternalBlobData* entry = registry_.CreateEntry(
+      content.uuid_, content.content_type_, content.content_disposition_);
+
+  // This flattens all blob references in the transportion content out and
+  // stores the complete item representation in the internal data.
+  BlobFlattener flattener(content, entry, &registry_);
+
+  // If we contain invalid references we're invalid input.
+  if (flattener.status == BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS) {
+    BreakAndFinishPendingBlob(content.uuid_,
+                              BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  // If we contain broken references (or we reference ourself) we want to fail.
+  if (flattener.status == BlobStatus::ERR_REFERENCED_BLOB_BROKEN) {
+    BreakAndFinishPendingBlob(content.uuid_,
+                              BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) {
+    BreakAndFinishPendingBlob(content.uuid_, BlobStatus::ERR_OUT_OF_MEMORY);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  entry->status_ = flattener.status;
+  entry->size_ = flattener.total_size;
+  std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry);
+
+  size_t num_building_dependent_blobs = 0;
+  std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs;
+  // We hold a handle to all blobs we're using. This is important, as our memory
+  // accounting can be delayed until OnEnoughSizeForBlobData is called, and we
+  // only free memory on canceling when we've done this accounting. If a
+  // dependent blob is dereferenced, then we're the last blob holding onto that
+  // data item, and we need to account for that. So we prevent that case by
+  // holding onto all blobs.
+  for (const std::pair<std::string, InternalBlobData*>& pending_blob :
+       flattener.dependent_blobs) {
+    dependent_blobs.push_back(
+        CreateHandle(pending_blob.first, pending_blob.second));
+    if (BlobStatusIsPending(pending_blob.second->status())) {
+      pending_blob.second->building_state_->build_completion_callbacks
+          .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished,
+                                ptr_factory_.GetWeakPtr(), content.uuid_));
+      num_building_dependent_blobs++;
     }
-    case BlobState::COMPLETE:
-      DCHECK(false) << "Blob already constructed: " << external_builder.uuid();
-      return;
   }
 
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->data->items().size());
-  UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken",
-                        entry->state == BlobState::BROKEN);
-  if (entry->state == BlobState::BROKEN) {
-    UMA_HISTOGRAM_ENUMERATION(
-        "Storage.Blob.BrokenReason", static_cast<int>(entry->broken_reason),
-        (static_cast<int>(IPCBlobCreationCancelCode::LAST) + 1));
+  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+         !can_populate_memory)
+      << "There is no pending content for the user to populate, so the "
+         "callback should be null.";
+  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+         can_populate_memory)
+      << "If we have pending content then there needs to be a callback "
+         "present.";
+
+  entry->building_state_.reset(new InternalBlobData::BuildingState(
+      /* transport_items_present */ !flattener.user_items.empty(),
+      /* user_data_population_callback */ can_populate_memory,
+      num_building_dependent_blobs,
+      /* memory_quota_needed */ flattener.memory_quota_needed > 0));

[michaeln, 2016/10/28 19:38:29]

probably you meant to cleanup these comments?

[dmurph, 2016/10/28 22:13:56]

I thought it made it more readable. Removed.

+
+  InternalBlobData::BuildingState* building_state =
+      entry->building_state_.get();
+  std::swap(building_state->copies, flattener.copies);
+  std::swap(building_state->dependent_blobs, dependent_blobs);
+  std::swap(building_state->user_items, flattener.user_items);
+
+  if (flattener.memory_quota_needed > 0) {
+    // We can finish our blob before returning from this method.
+    base::WeakPtr<QuotaAllocationTask> pending_request =
+        memory_controller_.ReserveMemoryQuota(
+            std::move(flattener.pending_memory_items),
+            base::Bind(&BlobStorageContext::OnEnoughSizeForMemory,
+                       ptr_factory_.GetWeakPtr(), content.uuid_));
+    if (entry->building_state_)
+      entry->building_state_->memory_quota_request = std::move(pending_request);
   }
-  size_t total_memory = 0, nonshared_memory = 0;
-  entry->data->GetMemoryUsage(&total_memory, &nonshared_memory);
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", total_memory / 1024);
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
-                       nonshared_memory / 1024);
-  TRACE_COUNTER1("Blob", "MemoryStoreUsageBytes", memory_usage_);
 
-  auto runner = base::ThreadTaskRunnerHandle::Get();
-  for (const auto& callback : entry->build_completion_callbacks) {
-    runner->PostTask(FROM_HERE,
-                     base::Bind(callback, entry->state == BlobState::COMPLETE,
-                                entry->broken_reason));
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+
+  return handle;
+}
+
+void BlobStorageContext::BreakAndFinishPendingBlob(const std::string& uuid,
+                                                   BlobStatus reason) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  DCHECK(entry);
+  DCHECK(BlobStatusIsError(reason));
+  PopulatationAllowedCallback user_data_population_callback;
+  if (entry->building_state_ &&
+      entry->building_state_->user_data_population_callback) {
+    user_data_population_callback =
+        entry->building_state_->user_data_population_callback;
+    entry->building_state_->user_data_population_callback.Reset();
+  }
+  ClearAndFreeMemory(uuid, entry);
+  entry->status_ = reason;
+  if (user_data_population_callback) {
+    user_data_population_callback.Run(
+        reason, std::vector<BlobMemoryController::FileCreationInfo>());
   }
-  entry->build_completion_callbacks.clear();
+  entry->items_.clear();
+  entry->offsets_.clear();
+  entry->size_ = 0;
+  FinishBuilding(entry);
 }
 
-void BlobStorageContext::CancelPendingBlob(const std::string& uuid,
-                                           IPCBlobCreationCancelCode reason) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  DCHECK(entry && entry->state == BlobState::PENDING);
-  entry->state = BlobState::BROKEN;
-  entry->broken_reason = reason;
-  CompletePendingBlob(BlobDataBuilder(uuid));
+void BlobStorageContext::FinishedPopulatingPendingBlob(
+    const std::string& uuid) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  CHECK(entry) << "There is no blob entry with uuid " << uuid;
+  DCHECK(BlobStatusIsPending(entry->status()));
+  FinishedPopulatingPendingBlob(entry);
 }
 
 void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+  InternalBlobData* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  ++(entry->refcount);
+  ++(entry->refcount_);
 }
 
 void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+  InternalBlobData* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  DCHECK_GT(entry->refcount, 0u);
-  if (--(entry->refcount) == 0) {
-    size_t memory_freeing = 0;
-    if (entry->state == BlobState::COMPLETE) {
-      memory_freeing = entry->data->GetUnsharedMemoryUsage();
-      entry->data->RemoveBlobFromShareableItems(uuid);
-    }
-    DCHECK_LE(memory_freeing, memory_usage_);
-    memory_usage_ -= memory_freeing;
+  DCHECK_GT(entry->refcount_, 0u);
+  if (--(entry->refcount_) == 0) {
+    ClearAndFreeMemory(uuid, entry);
     registry_.DeleteEntry(uuid);
   }
 }
@@ -202,263 +480,188 @@ void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
 std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot(
     const std::string& uuid) {
   std::unique_ptr<BlobDataSnapshot> result;
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (entry->state != BlobState::COMPLETE) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (entry->status() != BlobStatus::DONE)
     return result;
-  }
 
-  const InternalBlobData& data = *entry->data;
   std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot(
-      uuid, entry->content_type, entry->content_disposition));
-  snapshot->items_.reserve(data.items().size());
-  for (const auto& shareable_item : data.items()) {
+      uuid, entry->content_type(), entry->content_disposition()));
+  snapshot->items_.reserve(entry->items().size());
+  for (const auto& shareable_item : entry->items()) {
     snapshot->items_.push_back(shareable_item->item());
   }
+  memory_controller_.NotifyMemoryItemsUsed(entry->items());
   return snapshot;
 }
 
-bool BlobStorageContext::IsBroken(const std::string& uuid) const {
-  const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
-    return true;
-  }
-  return entry->state == BlobState::BROKEN;
+BlobStatus BlobStorageContext::GetBlobStatus(const std::string& uuid) const {
+  const InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (!entry)
+    return BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+  return entry->status();
 }
 
-bool BlobStorageContext::IsBeingBuilt(const std::string& uuid) const {
-  const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
-    return false;
+void BlobStorageContext::RunOnConstructionComplete(
+    const std::string& uuid,
+    const BlobStatusCallback& done) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  DCHECK(entry);
+  if (BlobStatusIsPending(entry->status())) {
+    entry->building_state_->build_completion_callbacks.push_back(done);
+    return;
   }
-  return entry->state == BlobState::PENDING;
+  done.Run(entry->status());
 }
 
-void BlobStorageContext::RunOnConstructionComplete(
+std::unique_ptr<BlobDataHandle> BlobStorageContext::CreateHandle(
     const std::string& uuid,
-    const BlobConstructedCallback& done) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+    InternalBlobData* entry) {
+  return base::WrapUnique(new BlobDataHandle(
+      uuid, entry->content_type_, entry->content_disposition_, entry->size_,
+      this, base::ThreadTaskRunnerHandle::Get().get()));
+}
+
+void BlobStorageContext::FinishedPopulatingPendingBlob(

[michaeln, 2016/10/28 19:38:29]

transporting naming convention? NotifyTransportComplete to balance
OnReadyForTransport?

[dmurph, 2016/10/28 22:13:56]

I used Population as the noun, let me know if you like Transport better.

[michaeln, 2016/11/07 21:47:04]

i do think sticking to 'transport' would help readers connect the dots more
easily

[dmurph, 2016/11/08 21:19:58]

Done.

+    InternalBlobData* entry) {
   DCHECK(entry);
-  switch (entry->state) {
-    case BlobState::COMPLETE:
-      done.Run(true, IPCBlobCreationCancelCode::UNKNOWN);
-      return;
-    case BlobState::BROKEN:
-      done.Run(false, entry->broken_reason);
-      return;
-    case BlobState::PENDING:
-      entry->build_completion_callbacks.push_back(done);
-      return;
-  }
-  NOTREACHED();
+  SetItemStateToPopulated(&entry->building_state_->user_items);
+  entry->status_ = BlobStatus::PENDING_INTERNALS;
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
 }
 
-bool BlobStorageContext::AppendAllocatedBlobItem(
-    const std::string& target_blob_uuid,
-    scoped_refptr<BlobDataItem> blob_item,
-    InternalBlobData::Builder* target_blob_builder,
-    IPCBlobCreationCancelCode* error_code) {
-  DCHECK(error_code);
-  *error_code = IPCBlobCreationCancelCode::UNKNOWN;
-  bool error = false;
-
-  // The blob data is stored in the canonical way which only contains a
-  // list of Data, File, and FileSystem items. Aggregated TYPE_BLOB items
-  // are expanded into the primitive constituent types and reused if possible.
-  // 1) The Data item is denoted by the raw data and length.
-  // 2) The File item is denoted by the file path, the range and the expected
-  //    modification time.
-  // 3) The FileSystem File item is denoted by the FileSystem URL, the range
-  //    and the expected modification time.
-  // 4) The Blob item is denoted by the source blob and an offset and size.
-  //    Internal items that are fully used by the new blob (not cut by the
-  //    offset or size) are shared between the blobs.  Otherwise, the relevant
-  //    portion of the item is copied.
-
-  DCHECK(blob_item->data_element_ptr());
-  const DataElement& data_element = blob_item->data_element();
-  uint64_t length = data_element.length();
-  uint64_t offset = data_element.offset();
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
-                       memory_usage_ / 1024);
-  switch (data_element.type()) {
-    case DataElement::TYPE_BYTES:
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024);
-      DCHECK(!offset);
-      if (memory_usage_ + length > kBlobStorageMaxMemoryUsage) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
-        break;
-      }
-      memory_usage_ += length;
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITH_QUOTA));
-      break;
-    case DataElement::TYPE_FILE: {
-      bool full_file = (length == std::numeric_limits<uint64_t>::max());
-      UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file);
-      if (!full_file) {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File",
-                             (length - offset) / 1024);
-      }
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_FILE_FILESYSTEM: {
-      bool full_file = (length == std::numeric_limits<uint64_t>::max());
-      UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown",
-                            full_file);
-      if (!full_file) {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem",
-                             (length - offset) / 1024);
-      }
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_BLOB: {
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob",
-                           (length - offset) / 1024);
-      // We grab the handle to ensure it stays around while we copy it.
-      std::unique_ptr<BlobDataHandle> src =
-          GetBlobDataFromUUID(data_element.blob_uuid());
-      if (!src || src->IsBroken() || src->IsBeingBuilt()) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN;
-        break;
-      }
-      BlobRegistryEntry* other_entry =
-          registry_.GetEntry(data_element.blob_uuid());
-      DCHECK(other_entry->data);
-      if (!AppendBlob(target_blob_uuid, *other_entry->data, offset, length,
-                      target_blob_builder)) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
+void BlobStorageContext::FinishBuilding(InternalBlobData* entry) {
+  DCHECK(entry);
+
+  if (BlobStatusIsPending(entry->status_)) {
+    for (const ItemCopyEntry& copy : entry->building_state_->copies) {
+      // Our source item can be a file if it was a slice of an unpopulated file,
+      // or a slice of data that was then paged to disk.
+      size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length());
+      DataElement::Type dest_type = copy.dest_item->item()->type();
+      switch (copy.source_item->item()->type()) {
+        case DataElement::TYPE_BYTES: {
+          DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION);
+          const char* src_data =
+              copy.source_item->item()->bytes() + copy.source_item_offset;
+          copy.dest_item->item()->item_->SetToBytes(src_data, dest_size);
+        } break;
+        case DataElement::TYPE_FILE:
+        case DataElement::TYPE_UNKNOWN:
+        case DataElement::TYPE_BLOB:
+        case DataElement::TYPE_BYTES_DESCRIPTION:
+        case DataElement::TYPE_FILE_FILESYSTEM:
+        case DataElement::TYPE_DISK_CACHE_ENTRY:
+          NOTREACHED();
+          break;
       }
-      break;
+      copy.dest_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
     }
-    case DataElement::TYPE_DISK_CACHE_ENTRY: {
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry",
-                           (length - offset) / 1024);
-      target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-          target_blob_uuid, blob_item,
-          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      break;
-    }
-    case DataElement::TYPE_BYTES_DESCRIPTION:
-    case DataElement::TYPE_UNKNOWN:
-      NOTREACHED();
-      break;
+    entry->status_ = BlobStatus::DONE;
+  }
+
+  std::vector<BlobStatusCallback> callbacks;
+  if (entry->building_state_.get()) {
+    std::swap(callbacks, entry->building_state_->build_completion_callbacks);
+    entry->building_state_.reset();
+  }
+
+  memory_controller_.NotifyMemoryItemsUsed(entry->items());
+
+  auto runner = base::ThreadTaskRunnerHandle::Get();
+  for (const auto& callback : callbacks)
+    runner->PostTask(FROM_HERE, base::Bind(callback, entry->status()));
+
+  for (const auto& shareable_item : entry->items()) {
+    DCHECK_NE(DataElement::TYPE_BYTES_DESCRIPTION,
+              shareable_item->item()->type());
+    DCHECK(shareable_item->IsPopulated()) << shareable_item->state();
   }
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend",
-                       memory_usage_ / 1024);
-  return !error;
 }
 
-bool BlobStorageContext::AppendBlob(
-    const std::string& target_blob_uuid,
-    const InternalBlobData& blob,
-    uint64_t offset,
-    uint64_t length,
-    InternalBlobData::Builder* target_blob_builder) {
-  DCHECK_GT(length, 0ull);
-
-  const std::vector<scoped_refptr<ShareableBlobDataItem>>& items = blob.items();
-  auto iter = items.begin();
-  if (offset) {
-    for (; iter != items.end(); ++iter) {
-      const BlobDataItem& item = *(iter->get()->item());
-      if (offset >= item.length())
-        offset -= item.length();
-      else
-        break;
-    }
+void BlobStorageContext::RequestUserPopulation(
+    InternalBlobData* entry,
+    std::vector<BlobMemoryController::FileCreationInfo> files) {
+  InternalBlobData::BuildingState* building_state =
+      entry->building_state_.get();
+  if (building_state->user_data_population_callback) {
+    PopulatationAllowedCallback user_data_population_callback =
+        building_state->user_data_population_callback;
+    building_state->user_data_population_callback.Reset();
+    user_data_population_callback.Run(BlobStatus::PENDING_TRANSPORT,
+                                      std::move(files));
+    return;
   }
+  DCHECK(files.empty());
+  FinishedPopulatingPendingBlob(entry);
+}
 
-  for (; iter != items.end() && length > 0; ++iter) {
-    scoped_refptr<ShareableBlobDataItem> shareable_item = iter->get();
-    const BlobDataItem& item = *(shareable_item->item());
-    uint64_t item_length = item.length();
-    DCHECK_GT(item_length, offset);
-    uint64_t current_length = item_length - offset;
-    uint64_t new_length = current_length > length ? length : current_length;
-
-    bool reusing_blob_item = offset == 0 && new_length == item.length();
-    UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item);
-    if (reusing_blob_item) {
-      shareable_item->referencing_blobs_mutable()->insert(target_blob_uuid);
-      target_blob_builder->AppendSharedBlobItem(shareable_item);
-      length -= new_length;
-      continue;
-    }
+void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid,
+                                               bool success) {
+  if (!success) {
+    BreakAndFinishPendingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+    return;
+  }
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (!entry || !entry->building_state_.get())
+    return;
+  InternalBlobData::BuildingState& building_state = *entry->building_state_;
+  DCHECK(!building_state.memory_quota_request);
+
+  if (building_state.transport_items_present) {
+    entry->status_ = BlobStatus::PENDING_TRANSPORT;
+    RequestUserPopulation(
+        entry, std::vector<BlobMemoryController::FileCreationInfo>());
+  } else {
+    entry->status_ = BlobStatus::PENDING_INTERNALS;
+  }
 
-    // We need to do copying of the items when we have a different offset or
-    // length
-    switch (item.type()) {
-      case DataElement::TYPE_BYTES: {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes",
-                             new_length / 1024);
-        if (memory_usage_ + new_length > kBlobStorageMaxMemoryUsage) {
-          return false;
-        }
-        DCHECK(!item.offset());
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToBytes(item.bytes() + offset,
-                            static_cast<int64_t>(new_length));
-        memory_usage_ += new_length;
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid, new BlobDataItem(std::move(element)),
-            ShareableBlobDataItem::POPULATED_WITH_QUOTA));
-      } break;
-      case DataElement::TYPE_FILE: {
-        DCHECK_NE(item.length(), std::numeric_limits<uint64_t>::max())
-            << "We cannot use a section of a file with an unknown length";
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File",
-                             new_length / 1024);
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToFilePathRange(item.path(), item.offset() + offset,
-                                    new_length,
-                                    item.expected_modification_time());
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid,
-            new BlobDataItem(std::move(element), item.data_handle_),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_FILE_FILESYSTEM: {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
-                             new_length / 1024);
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToFileSystemUrlRange(item.filesystem_url(),
-                                         item.offset() + offset, new_length,
-                                         item.expected_modification_time());
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid, new BlobDataItem(std::move(element)),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_DISK_CACHE_ENTRY: {
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToDiskCacheEntryRange(item.offset() + offset,
-                                          new_length);
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid,
-            new BlobDataItem(std::move(element), item.data_handle_,
-                             item.disk_cache_entry(),
-                             item.disk_cache_stream_index(),
-                             item.disk_cache_side_stream_index()),
-            ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
-      } break;
-      case DataElement::TYPE_BYTES_DESCRIPTION:
-      case DataElement::TYPE_BLOB:
-      case DataElement::TYPE_UNKNOWN:
-        CHECK(false) << "Illegal blob item type: " << item.type();
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::OnDependentBlobFinished(
+    const std::string& owning_blob_uuid,
+    BlobStatus status) {
+  InternalBlobData* entry = registry_.GetEntry(owning_blob_uuid);
+  if (!entry || !entry->building_state_)
+    return;
+
+  if (BlobStatusIsError(status)) {
+    DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status)
+        << "Referenced blob should never be dereferenced while we "
+        << "are depending on it, as our system holds a handle.";
+    BreakAndFinishPendingBlob(owning_blob_uuid,
+                              BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+    return;
+  }
+  DCHECK_GT(entry->building_state_->num_building_dependent_blobs, 0u);
+  --entry->building_state_->num_building_dependent_blobs;
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::ClearAndFreeMemory(const std::string& uuid,
+                                            InternalBlobData* entry) {
+  if (entry->building_state_) {
+    InternalBlobData::BuildingState* building_state =
+        entry->building_state_.get();
+    if (building_state->memory_quota_request) {
+      building_state->memory_quota_request->Cancel();
     }
-    length -= new_length;
-    offset = 0;
   }
-  return true;
+  entry->RemoveBlobFromShareableItems(uuid);
+  entry->items_.clear();
+  entry->offsets_.clear();
+}
+
+void BlobStorageContext::SetItemStateToPopulated(
+    std::vector<ShareableBlobDataItem*>* items) {
+  for (ShareableBlobDataItem* shareable_item : *items) {
+    DCHECK(shareable_item->state() == ShareableBlobDataItem::QUOTA_GRANTED);
+    shareable_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
+  }
 }
 
 }  // namespace storage