[BlobAsync] Disk support for blob storage

storage/browser/blob/blob_storage_context.cc

Index: storage/browser/blob/blob_storage_context.cc
diff --git a/storage/browser/blob/blob_storage_context.cc b/storage/browser/blob/blob_storage_context.cc
index 6ce45e53f90663cdda7d7b9d1293715ec92aabeb..1fc25d71f4fbedd0f25610b280201d88e728fb97 100644
--- a/storage/browser/blob/blob_storage_context.cc
+++ b/storage/browser/blob/blob_storage_context.cc
@@ -4,12 +4,10 @@
 
 #include "storage/browser/blob/blob_storage_context.h"
 
-#include <stddef.h>
-#include <stdint.h>
-
 #include <algorithm>
 #include <limits>
 #include <memory>
+#include <set>
 #include <utility>
 
 #include "base/bind.h"
@@ -19,57 +17,49 @@
 #include "base/memory/ptr_util.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "base/task_runner.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/trace_event/trace_event.h"
 #include "storage/browser/blob/blob_data_builder.h"
-#include "storage/browser/blob/blob_data_handle.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/blob_data_snapshot.h"
+#include "storage/browser/blob/blob_flattener.h"
+#include "storage/browser/blob/internal_blob_data.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
 #include "url/gurl.h"
 
 namespace storage {
-using BlobRegistryEntry = BlobStorageRegistry::Entry;
-using BlobState = BlobStorageRegistry::BlobState;
+using ItemCopyEntry = InternalBlobData::ItemCopyEntry;
 
-BlobStorageContext::BlobStorageContext() : memory_usage_(0) {}
+BlobStorageContext::BlobStorageContext() {}
 
 BlobStorageContext::~BlobStorageContext() {
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID(
     const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (!entry)
     return nullptr;
-  }
-  return base::WrapUnique(
-      new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
-                         this, base::ThreadTaskRunnerHandle::Get().get()));
+  return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL(
     const GURL& url) {
   std::string uuid;
-  BlobRegistryEntry* entry = registry_.GetEntryFromURL(url, &uuid);
-  if (!entry) {
+  InternalBlobData* entry = registry_.GetEntryFromURL(url, &uuid);
+  if (!entry)
     return nullptr;
-  }
-  return base::WrapUnique(
-      new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
-                         this, base::ThreadTaskRunnerHandle::Get().get()));
+  return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
     const BlobDataBuilder& external_builder) {
   TRACE_EVENT0("Blob", "Context::AddFinishedBlob");
-  CreatePendingBlob(external_builder.uuid(), external_builder.content_type_,
-                    external_builder.content_disposition_);
-  CompletePendingBlob(external_builder);
-  std::unique_ptr<BlobDataHandle> handle =
-      GetBlobDataFromUUID(external_builder.uuid_);
-  DecrementBlobRefCount(external_builder.uuid_);
-  return handle;
+  return BuildBlob(external_builder, PopulatationAllowedCallback());
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
@@ -78,123 +68,209 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
   return AddFinishedBlob(*builder);
 }
 
+std::unique_ptr<BlobDataHandle> BlobStorageContext::AddBrokenBlob(
+    const std::string& uuid,
+    const std::string& content_type,
+    const std::string& content_disposition,
+    BlobStatus reason) {
+  DCHECK(!registry_.HasEntry(uuid));
+  DCHECK(BlobStatusIsError(reason));
+  InternalBlobData* entry =
+      registry_.CreateEntry(uuid, content_type, content_disposition);
+  entry->status_ = reason;
+  FinishBuilding(entry);
+  return CreateHandle(uuid, entry);
+}
+
 bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url,
                                                const std::string& uuid) {
-  if (!registry_.CreateUrlMapping(blob_url, uuid)) {
+  if (!registry_.CreateUrlMapping(blob_url, uuid))
     return false;
-  }
   IncrementBlobRefCount(uuid);
   return true;
 }
 
 void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) {
   std::string uuid;
-  if (!registry_.DeleteURLMapping(blob_url, &uuid)) {
+  if (!registry_.DeleteURLMapping(blob_url, &uuid))
     return;
-  }
   DecrementBlobRefCount(uuid);
 }
 
-void BlobStorageContext::CreatePendingBlob(
-    const std::string& uuid,
-    const std::string& content_type,
-    const std::string& content_disposition) {
-  DCHECK(!registry_.GetEntry(uuid) && !uuid.empty());
-  registry_.CreateEntry(uuid, content_type, content_disposition);
+void BlobStorageContext::EnableDisk(
+    const base::FilePath& storage_directory,
+    scoped_refptr<base::TaskRunner> file_runner) {
+  memory_controller_.EnableDisk(storage_directory, std::move(file_runner));
 }
 
-void BlobStorageContext::CompletePendingBlob(
-    const BlobDataBuilder& external_builder) {
-  BlobRegistryEntry* entry = registry_.GetEntry(external_builder.uuid());
-  DCHECK(entry);
-  DCHECK(!entry->data.get()) << "Blob already constructed: "
-                             << external_builder.uuid();
-  // We want to handle storing our broken blob as well.
-  switch (entry->state) {
-    case BlobState::PENDING: {
-      entry->data_builder.reset(new InternalBlobData::Builder());
-      InternalBlobData::Builder* internal_data_builder =
-          entry->data_builder.get();
-
-      bool broken = false;
-      for (const auto& blob_item : external_builder.items_) {
-        IPCBlobCreationCancelCode error_code;
-        if (!AppendAllocatedBlobItem(external_builder.uuid_, blob_item,
-                                     internal_data_builder, &error_code)) {
-          broken = true;
-          memory_usage_ -= entry->data_builder->GetNonsharedMemoryUsage();
-          entry->state = BlobState::BROKEN;
-          entry->broken_reason = error_code;
-          entry->data_builder.reset(new InternalBlobData::Builder());
-          break;
-        }
-      }
-      entry->data = entry->data_builder->Build();
-      entry->data_builder.reset();
-      entry->state = broken ? BlobState::BROKEN : BlobState::COMPLETE;
-      break;
-    }
-    case BlobState::BROKEN: {
-      InternalBlobData::Builder builder;
-      entry->data = builder.Build();
-      break;
+std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
+    const BlobDataBuilder& content,
+    const PopulatationAllowedCallback& can_populate_memory) {
+  DCHECK(!registry_.HasEntry(content.uuid_));
+
+  InternalBlobData* entry = registry_.CreateEntry(
+      content.uuid_, content.content_type_, content.content_disposition_);
+
+  entry->building_state_.reset(new InternalBlobData::BuildingState());
+  InternalBlobData::BuildingState* building_state =
+      entry->building_state_.get();
+
+  // This flattens all blob references in the transportion content out and
+  // stores the complete item representation in the internal data.
+  BlobFlattener flattener(content, entry, &registry_);
+
+  // If we contain invalid references we're invalid input.
+  if (flattener.status == BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS) {
+    LOG(ERROR) << "invalid reference args";
+    BreakAndFinishPendingBlob(content.uuid_,
+                              BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  // If we contain broken references (or we reference ourself) we want to fail.
+  if (flattener.status == BlobStatus::ERR_REFERENCED_BLOB_BROKEN) {
+    LOG(ERROR) << "references don't exist";
+    BreakAndFinishPendingBlob(content.uuid_,
+                              BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  // We check to make sure that our memory calculations didn't overflow, which
+  // would mean that the user is trying to save more memory than is in the
+  // physical address space.
+  if (!flattener.transport_quota_needed.IsValid() ||
+      !flattener.copy_quota_needed.IsValid()) {
+    LOG(ERROR) << "sizes aren't valid..";
+    BreakAndFinishPendingBlob(content.uuid_,
+                              BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  uint64_t transport_quota_needed =
+      flattener.transport_quota_needed.ValueOrDie();
+  uint64_t copy_quota_needed = flattener.copy_quota_needed.ValueOrDie();
+  uint64_t quota_needed = transport_quota_needed + copy_quota_needed;
+
+  LOG(ERROR) << "We need " << transport_quota_needed
+             << " transportation bytes, and " << copy_quota_needed
+             << " copy bytes of quota.";
+
+  if (!memory_controller_.CanFitInSystem(quota_needed)) {
+    LOG(ERROR) << "we can't fit.";
+    BreakAndFinishPendingBlob(content.uuid_, BlobStatus::ERR_OUT_OF_MEMORY);
+    return CreateHandle(content.uuid_, entry);
+  }
+
+  // We store if we're waiting for the user to finish populating data in the
+  // |content| builder object.
+  if (flattener.status == BlobStatus::PENDING) {
+    entry->building_state_->data_pending_from_user = true;
+    building_state->ready_for_user_population_callback = can_populate_memory;
+  } else {
+    DCHECK_EQ(BlobStatus::DONE, flattener.status);
+    DCHECK(can_populate_memory.is_null());
+  }
+
+  entry->status_ = BlobStatus::PENDING;
+  entry->size_ = flattener.total_size.ValueOrDie();
+  entry->building_state_->transporting_items =
+      flattener.transport_pending_items;
+
+  std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry);
+
+  std::swap(building_state->copies, flattener.copies);
+
+  // We hold a handle to all blobs we're using. This is important, as our memory
+  // accounting can be delayed until OnEnoughSizeForBlobData is called, and we
+  // only free memory on canceling when we've done this accounting. If a
+  // dependent blob is dereferenced, then we're the last blob holding onto that
+  // data item, and we need to account for that. So we prevent that case by
+  // holding onto all blobs.
+  building_state->dependent_blobs_building = 0;
+  for (const std::pair<std::string, InternalBlobData*>& pending_blob :
+       flattener.dependent_blobs) {
+    building_state->dependent_blobs.push_back(
+        CreateHandle(pending_blob.first, pending_blob.second));
+    if (BlobStatusIsPending(pending_blob.second->status())) {
+      pending_blob.second->building_state_->build_completion_callbacks
+          .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished,
+                                AsWeakPtr(), content.uuid_));
+      building_state->dependent_blobs_building++;
     }
-    case BlobState::COMPLETE:
-      DCHECK(false) << "Blob already constructed: " << external_builder.uuid();
-      return;
   }
 
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->data->items().size());
-  UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken",
-                        entry->state == BlobState::BROKEN);
-  if (entry->state == BlobState::BROKEN) {
-    UMA_HISTOGRAM_ENUMERATION(
-        "Storage.Blob.BrokenReason", static_cast<int>(entry->broken_reason),
-        (static_cast<int>(IPCBlobCreationCancelCode::LAST) + 1));
+  entry->building_state_->user_quota_granted = transport_quota_needed == 0;
+  entry->building_state_->copy_quota_granted = copy_quota_needed == 0;
+
+  if (transport_quota_needed > 0) {
+    // We could finished our blob before returning from this method.
+    base::Optional<BlobMemoryController::PendingQuotaEntry> pending_entry =
+        memory_controller_.ReserveQuotaForItems(
+            std::move(flattener.transport_pending_items),
+            base::Bind(&BlobStorageContext::OnEnoughSizeForUserData,
+                       AsWeakPtr(), content.uuid_));
+    if (pending_entry)
+      entry->building_state_->user_quota_entry = pending_entry;
   }
-  size_t total_memory = 0, nonshared_memory = 0;
-  entry->data->GetMemoryUsage(&total_memory, &nonshared_memory);
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", total_memory / 1024);
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
-                       nonshared_memory / 1024);
-  TRACE_COUNTER1("Blob", "MemoryStoreUsageBytes", memory_usage_);
 
-  auto runner = base::ThreadTaskRunnerHandle::Get();
-  for (const auto& callback : entry->build_completion_callbacks) {
-    runner->PostTask(FROM_HERE,
-                     base::Bind(callback, entry->state == BlobState::COMPLETE,
-                                entry->broken_reason));
+  if (copy_quota_needed > 0) {
+    // We could finished our blob before returning from this method.
+    base::Optional<BlobMemoryController::PendingQuotaEntry> pending_entry =
+        memory_controller_.ReserveQuotaForItems(
+            std::move(flattener.copy_pending_items),
+            base::Bind(&BlobStorageContext::OnEnoughSizeForCopies, AsWeakPtr(),
+                       content.uuid_));
+    if (pending_entry)
+      entry->building_state_->copy_quota_entry = pending_entry;
+  }
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+
+  return handle;
+}
+
+void BlobStorageContext::BreakAndFinishPendingBlob(const std::string& uuid,
+                                                   BlobStatus reason) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  DCHECK(entry);
+  DCHECK(BlobStatusIsError(reason));
+  PopulatationAllowedCallback ready_for_population_callback =
+      std::move(entry->building_state_->ready_for_user_population_callback);
+  ClearAndFreeMemory(uuid, entry);
+  entry->status_ = reason;
+  if (!ready_for_population_callback.is_null()) {
+    ready_for_population_callback.Run(
+        reason, std::vector<BlobMemoryController::FileCreationInfo>());
   }
-  entry->build_completion_callbacks.clear();
+  entry->items_.clear();
+  entry->offsets_.clear();
+  entry->size_ = 0;
+  FinishBuilding(entry);
 }
 
-void BlobStorageContext::CancelPendingBlob(const std::string& uuid,
-                                           IPCBlobCreationCancelCode reason) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  DCHECK(entry && entry->state == BlobState::PENDING);
-  entry->state = BlobState::BROKEN;
-  entry->broken_reason = reason;
-  CompletePendingBlob(BlobDataBuilder(uuid));
+void BlobStorageContext::FinishedPopulatingPendingBlob(
+    const std::string& uuid) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  DCHECK_EQ(entry->status(), BlobStatus::PENDING);
+  entry->building_state_->data_pending_from_user = false;
+  SetItemStateToPopulated(&entry->building_state_->transporting_items);
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
 }
 
 void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+  InternalBlobData* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  ++(entry->refcount);
+  ++(entry->refcount_);
 }
 
 void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+  InternalBlobData* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  DCHECK_GT(entry->refcount, 0u);
-  if (--(entry->refcount) == 0) {
-    size_t memory_freeing = 0;
-    if (entry->state == BlobState::COMPLETE) {
-      memory_freeing = entry->data->GetUnsharedMemoryUsage();
-      entry->data->RemoveBlobFromShareableItems(uuid);
-    }
-    DCHECK_LE(memory_freeing, memory_usage_);
-    memory_usage_ -= memory_freeing;
+  DCHECK_GT(entry->refcount_, 0u);
+  if (--(entry->refcount_) == 0) {
+    ClearAndFreeMemory(uuid, entry);
     registry_.DeleteEntry(uuid);
   }
 }
@@ -202,255 +278,226 @@ void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
 std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot(
     const std::string& uuid) {
   std::unique_ptr<BlobDataSnapshot> result;
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (entry->state != BlobState::COMPLETE) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (entry->status() != BlobStatus::DONE)
     return result;
-  }
 
-  const InternalBlobData& data = *entry->data;
   std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot(
-      uuid, entry->content_type, entry->content_disposition));
-  snapshot->items_.reserve(data.items().size());
-  for (const auto& shareable_item : data.items()) {
+      uuid, entry->content_type(), entry->content_disposition()));
+  snapshot->items_.reserve(entry->items().size());
+  for (const auto& shareable_item : entry->items()) {
     snapshot->items_.push_back(shareable_item->item());
+    if (shareable_item->item()->type() == DataElement::TYPE_BYTES)
+      memory_controller_.UpdateBlobItemInRecents(shareable_item.get());
   }
   return snapshot;
 }
 
-bool BlobStorageContext::IsBroken(const std::string& uuid) const {
-  const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
-    return true;
-  }
-  return entry->state == BlobState::BROKEN;
-}
-
-bool BlobStorageContext::IsBeingBuilt(const std::string& uuid) const {
-  const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
-  if (!entry) {
-    return false;
-  }
-  return entry->state == BlobState::PENDING;
+BlobStatus BlobStorageContext::GetBlobStatus(const std::string& uuid) const {
+  const InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (!entry)
+    return BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+  return entry->status();
 }
 
 void BlobStorageContext::RunOnConstructionComplete(
     const std::string& uuid,
-    const BlobConstructedCallback& done) {
-  BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+    const BlobStatusCallback& done) {
+  InternalBlobData* entry = registry_.GetEntry(uuid);
   DCHECK(entry);
-  switch (entry->state) {
-    case BlobState::COMPLETE:
-      done.Run(true, IPCBlobCreationCancelCode::UNKNOWN);
-      return;
-    case BlobState::BROKEN:
-      done.Run(false, entry->broken_reason);
-      return;
-    case BlobState::PENDING:
-      entry->build_completion_callbacks.push_back(done);
-      return;
+  if (BlobStatusIsPending(entry->status())) {
+    entry->building_state_->build_completion_callbacks.push_back(done);
+    return;
   }
-  NOTREACHED();
+  done.Run(entry->status());
 }
 
-bool BlobStorageContext::AppendAllocatedBlobItem(
-    const std::string& target_blob_uuid,
-    scoped_refptr<BlobDataItem> blob_item,
-    InternalBlobData::Builder* target_blob_builder,
-    IPCBlobCreationCancelCode* error_code) {
-  DCHECK(error_code);
-  *error_code = IPCBlobCreationCancelCode::UNKNOWN;
-  bool error = false;
-
-  // The blob data is stored in the canonical way which only contains a
-  // list of Data, File, and FileSystem items. Aggregated TYPE_BLOB items
-  // are expanded into the primitive constituent types and reused if possible.
-  // 1) The Data item is denoted by the raw data and length.
-  // 2) The File item is denoted by the file path, the range and the expected
-  //    modification time.
-  // 3) The FileSystem File item is denoted by the FileSystem URL, the range
-  //    and the expected modification time.
-  // 4) The Blob item is denoted by the source blob and an offset and size.
-  //    Internal items that are fully used by the new blob (not cut by the
-  //    offset or size) are shared between the blobs.  Otherwise, the relevant
-  //    portion of the item is copied.
-
-  DCHECK(blob_item->data_element_ptr());
-  const DataElement& data_element = blob_item->data_element();
-  uint64_t length = data_element.length();
-  uint64_t offset = data_element.offset();
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
-                       memory_usage_ / 1024);
-  switch (data_element.type()) {
-    case DataElement::TYPE_BYTES:
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024);
-      DCHECK(!offset);
-      if (memory_usage_ + length > kBlobStorageMaxMemoryUsage) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
-        break;
-      }
-      memory_usage_ += length;
-      target_blob_builder->AppendSharedBlobItem(
-          new ShareableBlobDataItem(target_blob_uuid, blob_item));
-      break;
-    case DataElement::TYPE_FILE: {
-      bool full_file = (length == std::numeric_limits<uint64_t>::max());
-      UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file);
-      if (!full_file) {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File",
-                             (length - offset) / 1024);
-      }
-      target_blob_builder->AppendSharedBlobItem(
-          new ShareableBlobDataItem(target_blob_uuid, blob_item));
-      break;
-    }
-    case DataElement::TYPE_FILE_FILESYSTEM: {
-      bool full_file = (length == std::numeric_limits<uint64_t>::max());
-      UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown",
-                            full_file);
-      if (!full_file) {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem",
-                             (length - offset) / 1024);
-      }
-      target_blob_builder->AppendSharedBlobItem(
-          new ShareableBlobDataItem(target_blob_uuid, blob_item));
-      break;
-    }
-    case DataElement::TYPE_BLOB: {
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob",
-                           (length - offset) / 1024);
-      // We grab the handle to ensure it stays around while we copy it.
-      std::unique_ptr<BlobDataHandle> src =
-          GetBlobDataFromUUID(data_element.blob_uuid());
-      if (!src || src->IsBroken() || src->IsBeingBuilt()) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN;
-        break;
-      }
-      BlobRegistryEntry* other_entry =
-          registry_.GetEntry(data_element.blob_uuid());
-      DCHECK(other_entry->data);
-      if (!AppendBlob(target_blob_uuid, *other_entry->data, offset, length,
-                      target_blob_builder)) {
-        error = true;
-        *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
+std::unique_ptr<BlobDataHandle> BlobStorageContext::CreateHandle(
+    const std::string& uuid,
+    InternalBlobData* entry) {
+  return base::WrapUnique(new BlobDataHandle(
+      uuid, entry->content_type_, entry->content_disposition_, entry->size_,
+      this, base::ThreadTaskRunnerHandle::Get().get()));
+}
+
+void BlobStorageContext::FinishBuilding(InternalBlobData* entry) {
+  DCHECK(entry);
+
+  if (entry->status_ == BlobStatus::PENDING) {
+    for (const ItemCopyEntry& copy : entry->building_state_->copies) {
+      // Our source item can be a file if it was a slice of an unpopulated file,
+      // or a slice of data that was then paged to disk.
+      size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length());
+      DataElement::Type dest_type = copy.dest_item->item()->type();
+      switch (copy.source_item->item()->type()) {
+        case DataElement::TYPE_BYTES: {
+          DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION);
+          const char* src_data =
+              copy.source_item->item()->bytes() + copy.source_item_offset;
+          copy.dest_item->item()->item_->SetToBytes(src_data, dest_size);
+        } break;
+        case DataElement::TYPE_FILE: {
+          LOG(ERROR) << "Source item has been paged or we started as a file, "
+                        "so we're grabbing file ref "
+                     << dest_type;
+
+          // If we expected a memory item (and our source was paged to disk) we
+          // free that memory.
+          if (dest_type == DataElement::TYPE_BYTES_DESCRIPTION)
+            memory_controller_.FreeQuotaForPagedItemReference(copy.dest_item);
+
+          // We've been paged to disk, so free the memory of our temporary item,
+          // and create a new shared item with appropriate offset and length.
+          const DataElement& source_element =
+              copy.source_item->item()->data_element();
+          std::unique_ptr<DataElement> new_element(new DataElement());
+          new_element->SetToFilePathRange(
+              source_element.path(),
+              source_element.offset() + copy.source_item_offset, dest_size,
+              source_element.expected_modification_time());
+          scoped_refptr<BlobDataItem> new_item(new BlobDataItem(
+              std::move(new_element), copy.source_item->item()->data_handle()));
+          copy.dest_item->item_.swap(new_item);
+        } break;
+        case DataElement::TYPE_UNKNOWN:
+        case DataElement::TYPE_BLOB:
+        case DataElement::TYPE_BYTES_DESCRIPTION:
+        case DataElement::TYPE_FILE_FILESYSTEM:
+        case DataElement::TYPE_DISK_CACHE_ENTRY:
+          NOTREACHED();
+          break;
       }
-      break;
+      copy.dest_item->state_ = ShareableBlobDataItem::POPULATED_WITH_QUOTA;
     }
-    case DataElement::TYPE_DISK_CACHE_ENTRY: {
-      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry",
-                           (length - offset) / 1024);
-      target_blob_builder->AppendSharedBlobItem(
-          new ShareableBlobDataItem(target_blob_uuid, blob_item));
-      break;
+    entry->status_ = BlobStatus::DONE;
+  }
+  DCHECK(!BlobStatusIsPending(entry->status()));
+
+  LOG(ERROR) << "Finished blob.";
+
+  std::vector<BlobStatusCallback> callbacks;
+  if (entry->building_state_.get()) {
+    std::swap(callbacks, entry->building_state_->build_completion_callbacks);
+    entry->building_state_.reset();
+  }
+
+  auto runner = base::ThreadTaskRunnerHandle::Get();
+  for (const auto& callback : callbacks)
+    runner->PostTask(FROM_HERE, base::Bind(callback, entry->status()));
+
+  for (const auto& shareable_item : entry->items()) {
+    DCHECK_NE(DataElement::TYPE_BYTES_DESCRIPTION,
+              shareable_item->item()->type());
+    if (!shareable_item->IsPopulated()) {
+      PrintTo(*shareable_item, &LOG_STREAM(ERROR));
+      DCHECK(shareable_item->IsPopulated()) << shareable_item->state();
     }
-    case DataElement::TYPE_BYTES_DESCRIPTION:
-    case DataElement::TYPE_UNKNOWN:
-      NOTREACHED();
-      break;
+
+    if (shareable_item->item()->type() == DataElement::TYPE_BYTES)
+      memory_controller_.UpdateBlobItemInRecents(shareable_item.get());
   }
-  UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend",
-                       memory_usage_ / 1024);
-  return !error;
 }
 
-bool BlobStorageContext::AppendBlob(
-    const std::string& target_blob_uuid,
-    const InternalBlobData& blob,
-    uint64_t offset,
-    uint64_t length,
-    InternalBlobData::Builder* target_blob_builder) {
-  DCHECK_GT(length, 0ull);
-
-  const std::vector<scoped_refptr<ShareableBlobDataItem>>& items = blob.items();
-  auto iter = items.begin();
-  if (offset) {
-    for (; iter != items.end(); ++iter) {
-      const BlobDataItem& item = *(iter->get()->item());
-      if (offset >= item.length())
-        offset -= item.length();
-      else
-        break;
-    }
+void BlobStorageContext::OnEnoughSizeForCopies(

[michaeln, 2016/08/15 22:44:44]

since these are ReserveQuota completion callbacks, naming them to something
related sounding would help: OnQuotaReservedForXXXX

+    const std::string& uuid,
+    bool success,
+    std::vector<BlobMemoryController::FileCreationInfo> files) {
+  DCHECK(files.empty());
+  if (!success) {
+    BreakAndFinishPendingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+    return;
+  }
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (!entry || !entry->building_state_.get())
+    return;
+  entry->building_state_->copy_quota_granted = true;
+  entry->building_state_->copy_quota_entry = base::nullopt;
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::OnEnoughSizeForUserData(
+    const std::string& uuid,
+    bool success,
+    std::vector<BlobMemoryController::FileCreationInfo> files) {
+  if (!success) {
+    BreakAndFinishPendingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+    return;
+  }
+  InternalBlobData* entry = registry_.GetEntry(uuid);
+  if (!entry || !entry->building_state_.get())
+    return;
+  entry->building_state_->user_quota_granted = true;
+  entry->building_state_->user_quota_entry = base::nullopt;
+
+  if (entry->building_state_->data_pending_from_user) {
+    PopulatationAllowedCallback ready_for_user_population =
+        std::move(entry->building_state_->ready_for_user_population_callback);
+    DCHECK(!ready_for_user_population.is_null());
+    ready_for_user_population.Run(BlobStatus::PENDING, std::move(files));
+  } else {
+    DCHECK(files.empty());
+    SetItemStateToPopulated(&entry->building_state_->transporting_items);
   }
 
-  for (; iter != items.end() && length > 0; ++iter) {
-    scoped_refptr<ShareableBlobDataItem> shareable_item = iter->get();
-    const BlobDataItem& item = *(shareable_item->item());
-    uint64_t item_length = item.length();
-    DCHECK_GT(item_length, offset);
-    uint64_t current_length = item_length - offset;
-    uint64_t new_length = current_length > length ? length : current_length;
-
-    bool reusing_blob_item = offset == 0 && new_length == item.length();
-    UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item);
-    if (reusing_blob_item) {
-      shareable_item->referencing_blobs().insert(target_blob_uuid);
-      target_blob_builder->AppendSharedBlobItem(shareable_item);
-      length -= new_length;
-      continue;
-    }
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::OnDependentBlobFinished(
+    const std::string& owning_blob_uuid,
+    BlobStatus status) {
+  InternalBlobData* entry = registry_.GetEntry(owning_blob_uuid);
+  if (!entry)
+    return;
 
-    // We need to do copying of the items when we have a different offset or
-    // length
-    switch (item.type()) {
-      case DataElement::TYPE_BYTES: {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes",
-                             new_length / 1024);
-        if (memory_usage_ + new_length > kBlobStorageMaxMemoryUsage) {
-          return false;
-        }
-        DCHECK(!item.offset());
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToBytes(item.bytes() + offset,
-                            static_cast<int64_t>(new_length));
-        memory_usage_ += new_length;
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid, new BlobDataItem(std::move(element))));
-      } break;
-      case DataElement::TYPE_FILE: {
-        DCHECK_NE(item.length(), std::numeric_limits<uint64_t>::max())
-            << "We cannot use a section of a file with an unknown length";
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File",
-                             new_length / 1024);
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToFilePathRange(item.path(), item.offset() + offset,
-                                    new_length,
-                                    item.expected_modification_time());
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid,
-            new BlobDataItem(std::move(element), item.data_handle_)));
-      } break;
-      case DataElement::TYPE_FILE_FILESYSTEM: {
-        UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
-                             new_length / 1024);
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToFileSystemUrlRange(item.filesystem_url(),
-                                         item.offset() + offset, new_length,
-                                         item.expected_modification_time());
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid, new BlobDataItem(std::move(element))));
-      } break;
-      case DataElement::TYPE_DISK_CACHE_ENTRY: {
-        std::unique_ptr<DataElement> element(new DataElement());
-        element->SetToDiskCacheEntryRange(item.offset() + offset,
-                                          new_length);
-        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
-            target_blob_uuid,
-            new BlobDataItem(std::move(element), item.data_handle_,
-                             item.disk_cache_entry(),
-                             item.disk_cache_stream_index(),
-                             item.disk_cache_side_stream_index())));
-      } break;
-      case DataElement::TYPE_BYTES_DESCRIPTION:
-      case DataElement::TYPE_BLOB:
-      case DataElement::TYPE_UNKNOWN:
-        CHECK(false) << "Illegal blob item type: " << item.type();
+  if (BlobStatusIsError(status)) {
+    DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status)
+        << "Referenced blob should never be dereferenced while we "
+        << "are depending on it, as our system holds a handle.";
+    BreakAndFinishPendingBlob(owning_blob_uuid,
+                              BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+    return;
+  }
+  DCHECK_GT(entry->building_state_->dependent_blobs_building, 0u);
+  --entry->building_state_->dependent_blobs_building;
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::ClearAndFreeMemory(const std::string& uuid,
+                                            InternalBlobData* entry) {
+  if (entry->building_state_) {
+    if (entry->building_state_->user_quota_entry) {
+      memory_controller_.CancelQuotaReservation(
+          entry->building_state_->user_quota_entry.value());
+      entry->building_state_->user_quota_entry = base::nullopt;
+    }
+    if (entry->building_state_->copy_quota_entry) {
+      memory_controller_.CancelQuotaReservation(
+          entry->building_state_->copy_quota_entry.value());
+      entry->building_state_->copy_quota_entry = base::nullopt;
     }
-    length -= new_length;
-    offset = 0;
   }
-  return true;
+  entry->RemoveBlobFromShareableItems(uuid);
+  memory_controller_.MaybeFreeQuotaForItems(entry->items());
+  for (const auto& item_refptr : entry->items()) {
+    if (item_refptr->referencing_blobs().size() == 0)
+      memory_controller_.RemoveBlobItemInRecents(*item_refptr);
+  }
+
+  entry->items_.clear();
+  entry->offsets_.clear();
+}
+
+void BlobStorageContext::SetItemStateToPopulated(
+    std::vector<ShareableBlobDataItem*>* items) {
+  for (ShareableBlobDataItem* shareable_item : *items) {
+    DCHECK(shareable_item->state() == ShareableBlobDataItem::QUOTA_GRANTED);
+    shareable_item->state_ = ShareableBlobDataItem::POPULATED_WITH_QUOTA;
+  }
 }
 
 }  // namespace storage