Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1552)

Unified Diff: storage/browser/blob/blob_storage_context.cc

Issue 2448353002: [BlobAsync] Moving async handling into BlobStorageContext & quota out. (Closed)
Patch Set: comments from Marijn Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: storage/browser/blob/blob_storage_context.cc
diff --git a/storage/browser/blob/blob_storage_context.cc b/storage/browser/blob/blob_storage_context.cc
index fb1bc5b9f0d29a7a32a11c3d19b8e764d89eb3fb..a5ece241e32e38023368d55a700e490f92fa35fb 100644
--- a/storage/browser/blob/blob_storage_context.cc
+++ b/storage/browser/blob/blob_storage_context.cc
@@ -4,12 +4,10 @@
#include "storage/browser/blob/blob_storage_context.h"
-#include <stddef.h>
-#include <stdint.h>
-
#include <algorithm>
#include <limits>
#include <memory>
+#include <set>
#include <utility>
#include "base/bind.h"
@@ -19,57 +17,286 @@
#include "base/memory/ptr_util.h"
#include "base/message_loop/message_loop.h"
#include "base/metrics/histogram_macros.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "base/task_runner.h"
#include "base/threading/thread_task_runner_handle.h"
#include "base/trace_event/trace_event.h"
#include "storage/browser/blob/blob_data_builder.h"
-#include "storage/browser/blob/blob_data_handle.h"
#include "storage/browser/blob/blob_data_item.h"
#include "storage/browser/blob/blob_data_snapshot.h"
#include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
#include "url/gurl.h"
namespace storage {
-using BlobRegistryEntry = BlobStorageRegistry::Entry;
-using BlobState = BlobStorageRegistry::BlobState;
+namespace {
+using ItemCopyEntry = BlobEntry::ItemCopyEntry;
+using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+
+bool IsBytes(DataElement::Type type) {
+ return type == DataElement::TYPE_BYTES ||
+ type == DataElement::TYPE_BYTES_DESCRIPTION;
+}
+} // namespace
+
+BlobStorageContext::BlobFlattener::BlobFlattener(
+ const BlobDataBuilder& input_builder,
+ BlobEntry* output_blob,
+ BlobStorageRegistry* registry) {
+ const std::string& uuid = input_builder.uuid_;
+ std::set<std::string> dependent_blob_uuids;
+
+ size_t num_files_with_unknown_size = 0;
+ size_t num_building_dependent_blobs = 0;
+
+ base::CheckedNumeric<uint64_t> checked_total_size = 0;
+ base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0;
+
+ for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) {
+ const DataElement& input_element = input_item->data_element();
+ DataElement::Type type = input_element.type();
+ uint64_t length = input_element.length();
+
+ if (IsBytes(type)) {
+ DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length());
kinuko 2016/11/10 05:16:36 Is this "0 +" for type conversion?
dmurph 2016/11/10 19:53:19 I don't know why, but this make it link correctly.
+ checked_memory_quota_needed += length;
+ checked_total_size += length;
+ scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+ std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED);
+ pending_memory_items.push_back(item);
+ user_items.push_back(item.get());
+ output_blob->AppendSharedBlobItem(uuid, std::move(item));
+ continue;
+ }
+ if (type == DataElement::TYPE_BLOB) {
+ BlobEntry* ref_entry = registry->GetEntry(input_element.blob_uuid());
+
+ if (!ref_entry || input_element.blob_uuid() == uuid) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+
+ if (BlobStatusIsError(ref_entry->status())) {
+ status = BlobStatus::ERR_REFERENCED_BLOB_BROKEN;
+ return;
+ }
+
+ if (ref_entry->total_size() == DataElement::kUnknownSize) {
+ // We can't reference a blob with unknown size.
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+
+ if (dependent_blob_uuids.find(input_element.blob_uuid()) ==
+ dependent_blob_uuids.end()) {
+ dependent_blobs.push_back(
+ std::make_pair(input_element.blob_uuid(), ref_entry));
+ dependent_blob_uuids.insert(input_element.blob_uuid());
+ if (BlobStatusIsPending(ref_entry->status())) {
+ num_building_dependent_blobs++;
+ }
+ }
+
+ length = length == DataElement::kUnknownSize ? ref_entry->total_size()
+ : input_element.length();
+ checked_total_size += length;
+
+ // If we're referencing the whole blob, then we don't need to slice.
+ if (input_element.offset() == 0 && length == ref_entry->total_size()) {
+ for (const auto& shareable_item : ref_entry->items()) {
+ output_blob->AppendSharedBlobItem(uuid, shareable_item);
+ }
+ continue;
+ }
+
+ // Validate our reference has good offset & length.
+ if (input_element.offset() + length > ref_entry->total_size()) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+
+ BlobSlice slice(*ref_entry, input_element.offset(), length);
-BlobStorageContext::BlobStorageContext() : memory_usage_(0) {}
+ if (slice.first_source_item) {
+ copies.push_back(ItemCopyEntry(slice.first_source_item,
+ slice.first_item_slice_offset,
+ slice.dest_items.front()));
+ pending_memory_items.push_back(slice.dest_items.front());
+ }
+ if (slice.last_source_item) {
+ copies.push_back(
+ ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back()));
+ pending_memory_items.push_back(slice.dest_items.back());
+ }
+ checked_memory_quota_needed += slice.copying_memory_size;
+
+ for (auto& shareable_item : slice.dest_items) {
+ output_blob->AppendSharedBlobItem(uuid, std::move(shareable_item));
+ }
+ continue;
+ }
+
+ DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element))
+ << "File allocation not implemented.";
+ if (length == DataElement::kUnknownSize)
+ num_files_with_unknown_size++;
+
+ scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
+ std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
+
+ checked_total_size += length;
+ output_blob->AppendSharedBlobItem(uuid, std::move(item));
+ }
+
+ if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+ if (!checked_total_size.IsValid() || !checked_memory_quota_needed.IsValid()) {
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return;
+ }
+ total_size = checked_total_size.ValueOrDie();
+ memory_quota_needed = checked_memory_quota_needed.ValueOrDie();
+ if (memory_quota_needed) {
+ status = BlobStatus::PENDING_QUOTA;
+ } else {
+ status = BlobStatus::PENDING_INTERNALS;
+ }
+}
+
+BlobStorageContext::BlobFlattener::~BlobFlattener() {}
+
+BlobStorageContext::BlobSlice::BlobSlice(const BlobEntry& source,
+ uint64_t slice_offset,
+ uint64_t slice_size) {
+ const auto& source_items = source.items();
+ const auto& offsets = source.offsets();
+ DCHECK_LE(slice_offset + slice_size, source.total_size());
+ size_t item_index =
+ std::upper_bound(offsets.begin(), offsets.end(), slice_offset) -
+ offsets.begin();
+ uint64_t item_offset =
+ item_index == 0 ? slice_offset : slice_offset - offsets[item_index - 1];
+ size_t num_items = source_items.size();
+
+ size_t first_item_index = item_index;
+ copying_memory_size = 0;
+
+ // Read starting from 'first_item_index' and 'item_offset'.
+ for (uint64_t total_sliced = 0;
+ item_index < num_items && total_sliced < slice_size; item_index++) {
+ const scoped_refptr<BlobDataItem>& source_item =
+ source_items[item_index]->item();
+ uint64_t source_length = source_item->length();
+ DCHECK_NE(source_length, std::numeric_limits<uint64_t>::max());
+ DCHECK_NE(source_length, 0ull);
+
+ uint64_t read_size =
+ std::min(source_length - item_offset, slice_size - total_sliced);
+ total_sliced += read_size;
+
+ if (read_size == source_length) {
+ // We can share the entire item.
+ dest_items.push_back(source_items[item_index]);
+ continue;
+ }
+
+ scoped_refptr<BlobDataItem> data_item;
+ ShareableBlobDataItem::State state =
+ ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA;
+ switch (source_item->type()) {
+ case DataElement::TYPE_BYTES_DESCRIPTION:
+ case DataElement::TYPE_BYTES: {
+ if (item_index == first_item_index) {
+ first_item_slice_offset = item_offset;
+ first_source_item = source_items[item_index];
+ } else {
+ last_source_item = source_items[item_index];
+ }
+ copying_memory_size += read_size;
+ // Since we don't have quota yet for memory, we create temporary items
+ // for this data. When our blob is finished constructing, all dependent
+ // blobs are done, and we have enough memory quota, we'll copy the data
+ // over.
+ std::unique_ptr<DataElement> element(new DataElement());
+ element->SetToBytesDescription(base::checked_cast<size_t>(read_size));
+ data_item = new BlobDataItem(std::move(element));
+ state = ShareableBlobDataItem::QUOTA_NEEDED;
+ break;
+ }
+ case DataElement::TYPE_FILE: {
+ std::unique_ptr<DataElement> element(new DataElement());
+ element->SetToFilePathRange(
+ source_item->path(), source_item->offset() + item_offset, read_size,
+ source_item->expected_modification_time());
+ data_item =
+ new BlobDataItem(std::move(element), source_item->data_handle_);
+
+ DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element()))
+ << "File allocation unimplemented.";
+ break;
+ }
+ case DataElement::TYPE_FILE_FILESYSTEM: {
+ std::unique_ptr<DataElement> element(new DataElement());
+ element->SetToFileSystemUrlRange(
+ source_item->filesystem_url(), source_item->offset() + item_offset,
+ read_size, source_item->expected_modification_time());
+ data_item = new BlobDataItem(std::move(element));
+ break;
+ }
+ case DataElement::TYPE_DISK_CACHE_ENTRY: {
+ std::unique_ptr<DataElement> element(new DataElement());
+ element->SetToDiskCacheEntryRange(source_item->offset() + item_offset,
+ read_size);
+ data_item =
+ new BlobDataItem(std::move(element), source_item->data_handle_,
+ source_item->disk_cache_entry(),
+ source_item->disk_cache_stream_index(),
+ source_item->disk_cache_side_stream_index());
+ break;
+ }
+ case DataElement::TYPE_BLOB:
+ case DataElement::TYPE_UNKNOWN:
+ CHECK(false) << "Illegal blob item type: " << source_item->type();
+ }
+ dest_items.push_back(
+ new ShareableBlobDataItem(std::move(data_item), state));
+ item_offset = 0;
+ }
+}
+
+BlobStorageContext::BlobSlice::~BlobSlice() {}
+
+BlobStorageContext::BlobStorageContext()
+ : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()),
+ ptr_factory_(this) {}
BlobStorageContext::~BlobStorageContext() {
}
std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID(
const std::string& uuid) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- if (!entry) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ if (!entry)
return nullptr;
- }
- return base::WrapUnique(
- new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
- this, base::ThreadTaskRunnerHandle::Get().get()));
+ return CreateHandle(uuid, entry);
}
std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL(
const GURL& url) {
std::string uuid;
- BlobRegistryEntry* entry = registry_.GetEntryFromURL(url, &uuid);
- if (!entry) {
+ BlobEntry* entry = registry_.GetEntryFromURL(url, &uuid);
+ if (!entry)
return nullptr;
- }
- return base::WrapUnique(
- new BlobDataHandle(uuid, entry->content_type, entry->content_disposition,
- this, base::ThreadTaskRunnerHandle::Get().get()));
+ return CreateHandle(uuid, entry);
}
std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
const BlobDataBuilder& external_builder) {
TRACE_EVENT0("Blob", "Context::AddFinishedBlob");
- CreatePendingBlob(external_builder.uuid(), external_builder.content_type_,
- external_builder.content_disposition_);
- CompletePendingBlob(external_builder);
- std::unique_ptr<BlobDataHandle> handle =
- GetBlobDataFromUUID(external_builder.uuid_);
- DecrementBlobRefCount(external_builder.uuid_);
- return handle;
+ return BuildBlob(external_builder, TransportAllowedCallback());
}
std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
@@ -78,123 +305,167 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::AddFinishedBlob(
return AddFinishedBlob(*builder);
}
+std::unique_ptr<BlobDataHandle> BlobStorageContext::AddBrokenBlob(
+ const std::string& uuid,
+ const std::string& content_type,
+ const std::string& content_disposition,
+ BlobStatus reason) {
+ DCHECK(!registry_.HasEntry(uuid));
+ DCHECK(BlobStatusIsError(reason));
+ BlobEntry* entry =
+ registry_.CreateEntry(uuid, content_type, content_disposition);
+ entry->status_ = reason;
+ FinishBuilding(entry);
+ return CreateHandle(uuid, entry);
+}
+
bool BlobStorageContext::RegisterPublicBlobURL(const GURL& blob_url,
const std::string& uuid) {
- if (!registry_.CreateUrlMapping(blob_url, uuid)) {
+ if (!registry_.CreateUrlMapping(blob_url, uuid))
return false;
- }
IncrementBlobRefCount(uuid);
return true;
}
void BlobStorageContext::RevokePublicBlobURL(const GURL& blob_url) {
std::string uuid;
- if (!registry_.DeleteURLMapping(blob_url, &uuid)) {
+ if (!registry_.DeleteURLMapping(blob_url, &uuid))
return;
- }
DecrementBlobRefCount(uuid);
}
-void BlobStorageContext::CreatePendingBlob(
- const std::string& uuid,
- const std::string& content_type,
- const std::string& content_disposition) {
- DCHECK(!registry_.GetEntry(uuid) && !uuid.empty());
- registry_.CreateEntry(uuid, content_type, content_disposition);
-}
+std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
+ const BlobDataBuilder& content,
+ const TransportAllowedCallback& can_populate_memory) {
+ DCHECK(!registry_.HasEntry(content.uuid_));
-void BlobStorageContext::CompletePendingBlob(
- const BlobDataBuilder& external_builder) {
- BlobRegistryEntry* entry = registry_.GetEntry(external_builder.uuid());
- DCHECK(entry);
- DCHECK(!entry->data.get()) << "Blob already constructed: "
- << external_builder.uuid();
- // We want to handle storing our broken blob as well.
- switch (entry->state) {
- case BlobState::PENDING: {
- entry->data_builder.reset(new InternalBlobData::Builder());
- InternalBlobData::Builder* internal_data_builder =
- entry->data_builder.get();
-
- bool broken = false;
- for (const auto& blob_item : external_builder.items_) {
- IPCBlobCreationCancelCode error_code;
- if (!AppendAllocatedBlobItem(external_builder.uuid_, blob_item,
- internal_data_builder, &error_code)) {
- broken = true;
- memory_usage_ -= entry->data_builder->GetNonsharedMemoryUsage();
- entry->state = BlobState::BROKEN;
- entry->broken_reason = error_code;
- entry->data_builder.reset(new InternalBlobData::Builder());
- break;
- }
- }
- entry->data = entry->data_builder->Build();
- entry->data_builder.reset();
- entry->state = broken ? BlobState::BROKEN : BlobState::COMPLETE;
- break;
- }
- case BlobState::BROKEN: {
- InternalBlobData::Builder builder;
- entry->data = builder.Build();
- break;
+ BlobEntry* entry = registry_.CreateEntry(content.uuid_, content.content_type_,
+ content.content_disposition_);
+
+ // This flattens all blob references in the transportion content out and
+ // stores the complete item representation in the internal data.
+ BlobFlattener flattener(content, entry, &registry_);
+
+ // If we contain invalid references we're invalid input.
+ if (flattener.status == BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS) {
+ CancelBuildingBlob(content.uuid_,
+ BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS);
+ return CreateHandle(content.uuid_, entry);
+ }
+
+ // If we contain broken references (or we reference ourself) we want to fail.
+ if (flattener.status == BlobStatus::ERR_REFERENCED_BLOB_BROKEN) {
+ CancelBuildingBlob(content.uuid_, BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+ return CreateHandle(content.uuid_, entry);
+ }
+
+ if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) {
+ CancelBuildingBlob(content.uuid_, BlobStatus::ERR_OUT_OF_MEMORY);
+ return CreateHandle(content.uuid_, entry);
+ }
+
+ entry->status_ = flattener.status;
+ entry->size_ = flattener.total_size;
+ std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry);
+
+ size_t num_building_dependent_blobs = 0;
+ std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs;
+ // We hold a handle to all blobs we're using. This is important, as our memory
+ // accounting can be delayed until OnEnoughSizeForBlobData is called, and we
+ // only free memory on canceling when we've done this accounting. If a
+ // dependent blob is dereferenced, then we're the last blob holding onto that
+ // data item, and we need to account for that. So we prevent that case by
+ // holding onto all blobs.
+ for (const std::pair<std::string, BlobEntry*>& pending_blob :
+ flattener.dependent_blobs) {
+ dependent_blobs.push_back(
+ CreateHandle(pending_blob.first, pending_blob.second));
+ if (BlobStatusIsPending(pending_blob.second->status())) {
+ pending_blob.second->building_state_->build_completion_callbacks
+ .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished,
+ ptr_factory_.GetWeakPtr(), content.uuid_));
+ num_building_dependent_blobs++;
}
- case BlobState::COMPLETE:
- DCHECK(false) << "Blob already constructed: " << external_builder.uuid();
- return;
}
- UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->data->items().size());
- UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken",
- entry->state == BlobState::BROKEN);
- if (entry->state == BlobState::BROKEN) {
- UMA_HISTOGRAM_ENUMERATION(
- "Storage.Blob.BrokenReason", static_cast<int>(entry->broken_reason),
- (static_cast<int>(IPCBlobCreationCancelCode::LAST) + 1));
+ DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+ !can_populate_memory)
+ << "There is no pending content for the user to populate, so the "
+ "callback should be null.";
+ DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+ can_populate_memory)
+ << "If we have pending content then there needs to be a callback "
+ "present.";
+
+ entry->building_state_.reset(new BlobEntry::BuildingState(
+ !flattener.user_items.empty(), can_populate_memory,
+ num_building_dependent_blobs, flattener.memory_quota_needed > 0));
+
+ BlobEntry::BuildingState* building_state = entry->building_state_.get();
+ std::swap(building_state->copies, flattener.copies);
+ std::swap(building_state->dependent_blobs, dependent_blobs);
+ std::swap(building_state->user_items, flattener.user_items);
+
+ if (flattener.memory_quota_needed > 0) {
+ // We can finish our blob before returning from this method.
+ base::WeakPtr<QuotaAllocationTask> pending_request =
+ memory_controller_.ReserveMemoryQuota(
+ std::move(flattener.pending_memory_items),
+ base::Bind(&BlobStorageContext::OnEnoughSizeForMemory,
+ ptr_factory_.GetWeakPtr(), content.uuid_));
+ if (entry->building_state_)
kinuko 2016/11/10 05:16:36 When and why this could be null?
dmurph 2016/11/10 19:53:19 We can finish our blob before that method returns
+ entry->building_state_->memory_quota_request = std::move(pending_request);
}
- size_t total_memory = 0, nonshared_memory = 0;
- entry->data->GetMemoryUsage(&total_memory, &nonshared_memory);
- UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", total_memory / 1024);
- UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
- nonshared_memory / 1024);
- TRACE_COUNTER1("Blob", "MemoryStoreUsageBytes", memory_usage_);
- auto runner = base::ThreadTaskRunnerHandle::Get();
- for (const auto& callback : entry->build_completion_callbacks) {
- runner->PostTask(FROM_HERE,
- base::Bind(callback, entry->state == BlobState::COMPLETE,
- entry->broken_reason));
+ if (entry->CanFinishBuilding())
+ FinishBuilding(entry);
+
+ return handle;
+}
+
+void BlobStorageContext::CancelBuildingBlob(const std::string& uuid,
+ BlobStatus reason) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ DCHECK(entry);
+ DCHECK(BlobStatusIsError(reason));
+ TransportAllowedCallback user_data_population_callback;
+ if (entry->building_state_ &&
+ entry->building_state_->transport_allowed_callback) {
+ user_data_population_callback =
+ entry->building_state_->transport_allowed_callback;
+ entry->building_state_->transport_allowed_callback.Reset();
}
- entry->build_completion_callbacks.clear();
+ ClearAndFreeMemory(uuid, entry);
+ entry->status_ = reason;
+ if (user_data_population_callback) {
+ user_data_population_callback.Run(
+ reason, std::vector<BlobMemoryController::FileCreationInfo>());
+ }
+ entry->items_.clear();
+ entry->offsets_.clear();
+ entry->size_ = 0;
+ FinishBuilding(entry);
}
-void BlobStorageContext::CancelPendingBlob(const std::string& uuid,
- IPCBlobCreationCancelCode reason) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- DCHECK(entry && entry->state == BlobState::PENDING);
- entry->state = BlobState::BROKEN;
- entry->broken_reason = reason;
- CompletePendingBlob(BlobDataBuilder(uuid));
+void BlobStorageContext::NotifyTransportComplete(const std::string& uuid) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ CHECK(entry) << "There is no blob entry with uuid " << uuid;
+ DCHECK(BlobStatusIsPending(entry->status()));
+ NotifyTransportComplete(entry);
}
void BlobStorageContext::IncrementBlobRefCount(const std::string& uuid) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+ BlobEntry* entry = registry_.GetEntry(uuid);
DCHECK(entry);
- ++(entry->refcount);
+ ++(entry->refcount_);
}
void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+ BlobEntry* entry = registry_.GetEntry(uuid);
DCHECK(entry);
- DCHECK_GT(entry->refcount, 0u);
- if (--(entry->refcount) == 0) {
- size_t memory_freeing = 0;
- if (entry->state == BlobState::COMPLETE) {
- memory_freeing = entry->data->GetUnsharedMemoryUsage();
- entry->data->RemoveBlobFromShareableItems(uuid);
- }
- DCHECK_LE(memory_freeing, memory_usage_);
- memory_usage_ -= memory_freeing;
+ DCHECK_GT(entry->refcount_, 0u);
+ if (--(entry->refcount_) == 0) {
+ ClearAndFreeMemory(uuid, entry);
registry_.DeleteEntry(uuid);
}
}
@@ -202,263 +473,185 @@ void BlobStorageContext::DecrementBlobRefCount(const std::string& uuid) {
std::unique_ptr<BlobDataSnapshot> BlobStorageContext::CreateSnapshot(
const std::string& uuid) {
std::unique_ptr<BlobDataSnapshot> result;
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- if (entry->state != BlobState::COMPLETE) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ if (entry->status() != BlobStatus::DONE)
return result;
- }
- const InternalBlobData& data = *entry->data;
std::unique_ptr<BlobDataSnapshot> snapshot(new BlobDataSnapshot(
- uuid, entry->content_type, entry->content_disposition));
- snapshot->items_.reserve(data.items().size());
- for (const auto& shareable_item : data.items()) {
+ uuid, entry->content_type(), entry->content_disposition()));
+ snapshot->items_.reserve(entry->items().size());
+ for (const auto& shareable_item : entry->items()) {
snapshot->items_.push_back(shareable_item->item());
}
+ memory_controller_.NotifyMemoryItemsUsed(entry->items());
return snapshot;
}
-bool BlobStorageContext::IsBroken(const std::string& uuid) const {
- const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- if (!entry) {
- return true;
- }
- return entry->state == BlobState::BROKEN;
+BlobStatus BlobStorageContext::GetBlobStatus(const std::string& uuid) const {
+ const BlobEntry* entry = registry_.GetEntry(uuid);
+ if (!entry)
+ return BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+ return entry->status();
}
-bool BlobStorageContext::IsBeingBuilt(const std::string& uuid) const {
- const BlobRegistryEntry* entry = registry_.GetEntry(uuid);
- if (!entry) {
- return false;
+void BlobStorageContext::RunOnConstructionComplete(
+ const std::string& uuid,
+ const BlobStatusCallback& done) {
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ DCHECK(entry);
+ if (BlobStatusIsPending(entry->status())) {
+ entry->building_state_->build_completion_callbacks.push_back(done);
+ return;
}
- return entry->state == BlobState::PENDING;
+ done.Run(entry->status());
}
-void BlobStorageContext::RunOnConstructionComplete(
+std::unique_ptr<BlobDataHandle> BlobStorageContext::CreateHandle(
const std::string& uuid,
- const BlobConstructedCallback& done) {
- BlobRegistryEntry* entry = registry_.GetEntry(uuid);
+ BlobEntry* entry) {
+ return base::WrapUnique(new BlobDataHandle(
+ uuid, entry->content_type_, entry->content_disposition_, entry->size_,
+ this, base::ThreadTaskRunnerHandle::Get().get()));
+}
+
+void BlobStorageContext::NotifyTransportComplete(BlobEntry* entry) {
DCHECK(entry);
- switch (entry->state) {
- case BlobState::COMPLETE:
- done.Run(true, IPCBlobCreationCancelCode::UNKNOWN);
- return;
- case BlobState::BROKEN:
- done.Run(false, entry->broken_reason);
- return;
- case BlobState::PENDING:
- entry->build_completion_callbacks.push_back(done);
- return;
- }
- NOTREACHED();
+ SetItemStateToPopulated(&entry->building_state_->user_items);
+ entry->status_ = BlobStatus::PENDING_INTERNALS;
+ if (entry->CanFinishBuilding())
+ FinishBuilding(entry);
}
-bool BlobStorageContext::AppendAllocatedBlobItem(
- const std::string& target_blob_uuid,
- scoped_refptr<BlobDataItem> blob_item,
- InternalBlobData::Builder* target_blob_builder,
- IPCBlobCreationCancelCode* error_code) {
- DCHECK(error_code);
- *error_code = IPCBlobCreationCancelCode::UNKNOWN;
- bool error = false;
-
- // The blob data is stored in the canonical way which only contains a
- // list of Data, File, and FileSystem items. Aggregated TYPE_BLOB items
- // are expanded into the primitive constituent types and reused if possible.
- // 1) The Data item is denoted by the raw data and length.
- // 2) The File item is denoted by the file path, the range and the expected
- // modification time.
- // 3) The FileSystem File item is denoted by the FileSystem URL, the range
- // and the expected modification time.
- // 4) The Blob item is denoted by the source blob and an offset and size.
- // Internal items that are fully used by the new blob (not cut by the
- // offset or size) are shared between the blobs. Otherwise, the relevant
- // portion of the item is copied.
-
- DCHECK(blob_item->data_element_ptr());
- const DataElement& data_element = blob_item->data_element();
- uint64_t length = data_element.length();
- uint64_t offset = data_element.offset();
- UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
- memory_usage_ / 1024);
- switch (data_element.type()) {
- case DataElement::TYPE_BYTES:
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Bytes", length / 1024);
- DCHECK(!offset);
- if (memory_usage_ + length > kBlobStorageMaxMemoryUsage) {
- error = true;
- *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
- break;
- }
- memory_usage_ += length;
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid, blob_item,
- ShareableBlobDataItem::POPULATED_WITH_QUOTA));
- break;
- case DataElement::TYPE_FILE: {
- bool full_file = (length == std::numeric_limits<uint64_t>::max());
- UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.File.Unknown", full_file);
- if (!full_file) {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.File",
- (length - offset) / 1024);
- }
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid, blob_item,
- ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
- break;
- }
- case DataElement::TYPE_FILE_FILESYSTEM: {
- bool full_file = (length == std::numeric_limits<uint64_t>::max());
- UMA_HISTOGRAM_BOOLEAN("Storage.BlobItemSize.FileSystem.Unknown",
- full_file);
- if (!full_file) {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.FileSystem",
- (length - offset) / 1024);
- }
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid, blob_item,
- ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
- break;
- }
- case DataElement::TYPE_BLOB: {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.Blob",
- (length - offset) / 1024);
- // We grab the handle to ensure it stays around while we copy it.
- std::unique_ptr<BlobDataHandle> src =
- GetBlobDataFromUUID(data_element.blob_uuid());
- if (!src || src->IsBroken() || src->IsBeingBuilt()) {
- error = true;
- *error_code = IPCBlobCreationCancelCode::REFERENCED_BLOB_BROKEN;
- break;
- }
- BlobRegistryEntry* other_entry =
- registry_.GetEntry(data_element.blob_uuid());
- DCHECK(other_entry->data);
- if (!AppendBlob(target_blob_uuid, *other_entry->data, offset, length,
- target_blob_builder)) {
- error = true;
- *error_code = IPCBlobCreationCancelCode::OUT_OF_MEMORY;
+void BlobStorageContext::FinishBuilding(BlobEntry* entry) {
+ DCHECK(entry);
+
+ if (BlobStatusIsPending(entry->status_)) {
kinuko 2016/11/10 05:16:36 Is PENDING_INTERNALS used mostly for this if block
dmurph 2016/11/10 19:53:19 Yes. We're still not done if we haven't copied ove
+ for (const ItemCopyEntry& copy : entry->building_state_->copies) {
+ // Our source item can be a file if it was a slice of an unpopulated file,
+ // or a slice of data that was then paged to disk.
+ size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length());
+ DataElement::Type dest_type = copy.dest_item->item()->type();
+ switch (copy.source_item->item()->type()) {
+ case DataElement::TYPE_BYTES: {
+ DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION);
+ const char* src_data =
+ copy.source_item->item()->bytes() + copy.source_item_offset;
+ copy.dest_item->item()->item_->SetToBytes(src_data, dest_size);
+ } break;
+ case DataElement::TYPE_FILE:
+ case DataElement::TYPE_UNKNOWN:
+ case DataElement::TYPE_BLOB:
+ case DataElement::TYPE_BYTES_DESCRIPTION:
+ case DataElement::TYPE_FILE_FILESYSTEM:
+ case DataElement::TYPE_DISK_CACHE_ENTRY:
+ NOTREACHED();
+ break;
}
- break;
+ copy.dest_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
}
- case DataElement::TYPE_DISK_CACHE_ENTRY: {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry",
- (length - offset) / 1024);
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid, blob_item,
- ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
- break;
- }
- case DataElement::TYPE_BYTES_DESCRIPTION:
- case DataElement::TYPE_UNKNOWN:
- NOTREACHED();
- break;
+ entry->status_ = BlobStatus::DONE;
+ }
+
+ std::vector<BlobStatusCallback> callbacks;
+ if (entry->building_state_.get()) {
+ std::swap(callbacks, entry->building_state_->build_completion_callbacks);
+ entry->building_state_.reset();
+ }
+
+ memory_controller_.NotifyMemoryItemsUsed(entry->items());
+
+ auto runner = base::ThreadTaskRunnerHandle::Get();
+ for (const auto& callback : callbacks)
+ runner->PostTask(FROM_HERE, base::Bind(callback, entry->status()));
+
+ for (const auto& shareable_item : entry->items()) {
+ DCHECK_NE(DataElement::TYPE_BYTES_DESCRIPTION,
+ shareable_item->item()->type());
+ DCHECK(shareable_item->IsPopulated()) << shareable_item->state();
}
- UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeAfterAppend",
- memory_usage_ / 1024);
- return !error;
}
-bool BlobStorageContext::AppendBlob(
- const std::string& target_blob_uuid,
- const InternalBlobData& blob,
- uint64_t offset,
- uint64_t length,
- InternalBlobData::Builder* target_blob_builder) {
- DCHECK_GT(length, 0ull);
-
- const std::vector<scoped_refptr<ShareableBlobDataItem>>& items = blob.items();
- auto iter = items.begin();
- if (offset) {
- for (; iter != items.end(); ++iter) {
- const BlobDataItem& item = *(iter->get()->item());
- if (offset >= item.length())
- offset -= item.length();
- else
- break;
- }
+void BlobStorageContext::RequestTransport(
+ BlobEntry* entry,
+ std::vector<BlobMemoryController::FileCreationInfo> files) {
+ BlobEntry::BuildingState* building_state = entry->building_state_.get();
+ if (building_state->transport_allowed_callback) {
+ TransportAllowedCallback user_data_population_callback =
+ building_state->transport_allowed_callback;
+ building_state->transport_allowed_callback.Reset();
+ user_data_population_callback.Run(BlobStatus::PENDING_TRANSPORT,
+ std::move(files));
+ return;
}
+ DCHECK(files.empty());
+ NotifyTransportComplete(entry);
+}
- for (; iter != items.end() && length > 0; ++iter) {
- scoped_refptr<ShareableBlobDataItem> shareable_item = iter->get();
- const BlobDataItem& item = *(shareable_item->item());
- uint64_t item_length = item.length();
- DCHECK_GT(item_length, offset);
- uint64_t current_length = item_length - offset;
- uint64_t new_length = current_length > length ? length : current_length;
-
- bool reusing_blob_item = offset == 0 && new_length == item.length();
- UMA_HISTOGRAM_BOOLEAN("Storage.Blob.ReusedItem", reusing_blob_item);
- if (reusing_blob_item) {
- shareable_item->referencing_blobs_mutable()->insert(target_blob_uuid);
- target_blob_builder->AppendSharedBlobItem(shareable_item);
- length -= new_length;
- continue;
- }
+void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid,
+ bool success) {
+ if (!success) {
+ CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+ return;
+ }
+ BlobEntry* entry = registry_.GetEntry(uuid);
+ if (!entry || !entry->building_state_.get())
+ return;
+ BlobEntry::BuildingState& building_state = *entry->building_state_;
+ DCHECK(!building_state.memory_quota_request);
+
+ if (building_state.transport_items_present) {
+ entry->status_ = BlobStatus::PENDING_TRANSPORT;
+ RequestTransport(entry,
+ std::vector<BlobMemoryController::FileCreationInfo>());
+ } else {
+ entry->status_ = BlobStatus::PENDING_INTERNALS;
+ }
- // We need to do copying of the items when we have a different offset or
- // length
- switch (item.type()) {
- case DataElement::TYPE_BYTES: {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.Bytes",
- new_length / 1024);
- if (memory_usage_ + new_length > kBlobStorageMaxMemoryUsage) {
- return false;
- }
- DCHECK(!item.offset());
- std::unique_ptr<DataElement> element(new DataElement());
- element->SetToBytes(item.bytes() + offset,
- static_cast<int64_t>(new_length));
- memory_usage_ += new_length;
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid, new BlobDataItem(std::move(element)),
- ShareableBlobDataItem::POPULATED_WITH_QUOTA));
- } break;
- case DataElement::TYPE_FILE: {
- DCHECK_NE(item.length(), std::numeric_limits<uint64_t>::max())
- << "We cannot use a section of a file with an unknown length";
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File",
- new_length / 1024);
- std::unique_ptr<DataElement> element(new DataElement());
- element->SetToFilePathRange(item.path(), item.offset() + offset,
- new_length,
- item.expected_modification_time());
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid,
- new BlobDataItem(std::move(element), item.data_handle_),
- ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
- } break;
- case DataElement::TYPE_FILE_FILESYSTEM: {
- UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
- new_length / 1024);
- std::unique_ptr<DataElement> element(new DataElement());
- element->SetToFileSystemUrlRange(item.filesystem_url(),
- item.offset() + offset, new_length,
- item.expected_modification_time());
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid, new BlobDataItem(std::move(element)),
- ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
- } break;
- case DataElement::TYPE_DISK_CACHE_ENTRY: {
- std::unique_ptr<DataElement> element(new DataElement());
- element->SetToDiskCacheEntryRange(item.offset() + offset,
- new_length);
- target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
- target_blob_uuid,
- new BlobDataItem(std::move(element), item.data_handle_,
- item.disk_cache_entry(),
- item.disk_cache_stream_index(),
- item.disk_cache_side_stream_index()),
- ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA));
- } break;
- case DataElement::TYPE_BYTES_DESCRIPTION:
- case DataElement::TYPE_BLOB:
- case DataElement::TYPE_UNKNOWN:
- CHECK(false) << "Illegal blob item type: " << item.type();
+ if (entry->CanFinishBuilding())
+ FinishBuilding(entry);
+}
+
+void BlobStorageContext::OnDependentBlobFinished(
+ const std::string& owning_blob_uuid,
+ BlobStatus status) {
+ BlobEntry* entry = registry_.GetEntry(owning_blob_uuid);
+ if (!entry || !entry->building_state_)
+ return;
+
+ if (BlobStatusIsError(status)) {
+ DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status)
+ << "Referenced blob should never be dereferenced while we "
+ << "are depending on it, as our system holds a handle.";
+ CancelBuildingBlob(owning_blob_uuid,
+ BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
+ return;
+ }
+ DCHECK_GT(entry->building_state_->num_building_dependent_blobs, 0u);
+ --entry->building_state_->num_building_dependent_blobs;
+
+ if (entry->CanFinishBuilding())
+ FinishBuilding(entry);
+}
+
+void BlobStorageContext::ClearAndFreeMemory(const std::string& uuid,
+ BlobEntry* entry) {
+ if (entry->building_state_) {
+ BlobEntry::BuildingState* building_state = entry->building_state_.get();
+ if (building_state->memory_quota_request) {
+ building_state->memory_quota_request->Cancel();
}
- length -= new_length;
- offset = 0;
}
- return true;
+ entry->RemoveBlobFromShareableItems(uuid);
+ entry->items_.clear();
+ entry->offsets_.clear();
+}
+
+void BlobStorageContext::SetItemStateToPopulated(
+ std::vector<ShareableBlobDataItem*>* items) {
+ for (ShareableBlobDataItem* shareable_item : *items) {
kinuko 2016/11/10 05:16:36 nit: I'd probably just put this 4-lines loop direc
dmurph 2016/11/10 19:53:19 Done.
+ DCHECK(shareable_item->state() == ShareableBlobDataItem::QUOTA_GRANTED);
+ shareable_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
+ }
}
} // namespace storage

Powered by Google App Engine
This is Rietveld 408576698