Index: storage/browser/blob/blob_storage_context.cc |
diff --git a/storage/browser/blob/blob_storage_context.cc b/storage/browser/blob/blob_storage_context.cc |
index 1fecda89f5b6799136cab3bc4089840251ad8f3b..ef81ab18c80e00bc58c7cd291542c1bea9988409 100644 |
--- a/storage/browser/blob/blob_storage_context.cc |
+++ b/storage/browser/blob/blob_storage_context.cc |
@@ -92,9 +92,13 @@ BlobStorageContext::BlobFlattener::BlobFlattener( |
size_t num_files_with_unknown_size = 0; |
size_t num_building_dependent_blobs = 0; |
+ bool found_memory_transport = false; |
+ bool found_file_transport = false; |
+ |
base::CheckedNumeric<uint64_t> checked_total_size = 0; |
base::CheckedNumeric<uint64_t> checked_total_memory_size = 0; |
- base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0; |
+ base::CheckedNumeric<uint64_t> checked_transport_quota_needed = 0; |
+ base::CheckedNumeric<uint64_t> checked_copy_quota_needed = 0; |
for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) { |
const DataElement& input_element = input_item->data_element(); |
@@ -105,11 +109,19 @@ BlobStorageContext::BlobFlattener::BlobFlattener( |
if (IsBytes(type)) { |
DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length()); |
- checked_memory_quota_needed += length; |
+ found_memory_transport = true; |
+ if (found_file_transport) { |
+ // We cannot have both memory and file transport items. |
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
+ return; |
+ } |
+ contains_unpopulated_transport_items |= |
+ (type == DataElement::TYPE_BYTES_DESCRIPTION); |
+ checked_transport_quota_needed += length; |
checked_total_size += length; |
scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem( |
std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED); |
- pending_memory_items.push_back(item); |
+ pending_transport_items.push_back(item); |
transport_items.push_back(item.get()); |
output_blob->AppendSharedBlobItem(std::move(item)); |
continue; |
@@ -174,14 +186,14 @@ BlobStorageContext::BlobFlattener::BlobFlattener( |
copies.push_back(ItemCopyEntry(slice.first_source_item, |
slice.first_item_slice_offset, |
slice.dest_items.front())); |
- pending_memory_items.push_back(slice.dest_items.front()); |
+ pending_copy_items.push_back(slice.dest_items.front()); |
} |
if (slice.last_source_item) { |
copies.push_back( |
ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back())); |
- pending_memory_items.push_back(slice.dest_items.back()); |
+ pending_copy_items.push_back(slice.dest_items.back()); |
} |
- checked_memory_quota_needed += slice.copying_memory_size; |
+ checked_copy_quota_needed += slice.copying_memory_size; |
for (auto& shareable_item : slice.dest_items) { |
output_blob->AppendSharedBlobItem(std::move(shareable_item)); |
@@ -189,14 +201,30 @@ BlobStorageContext::BlobFlattener::BlobFlattener( |
continue; |
} |
- DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element)) |
- << "File allocation not implemented."; |
+ // If the source item is a temporary file item, then we need to keep track |
+ // of that and mark it as needing quota. |
+ scoped_refptr<ShareableBlobDataItem> item; |
+ if (BlobDataBuilder::IsFutureFileItem(input_element)) { |
+ found_file_transport = true; |
+ if (found_memory_transport) { |
+ // We cannot have both memory and file transport items. |
+ status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
+ return; |
+ } |
+ contains_unpopulated_transport_items = true; |
+ item = new ShareableBlobDataItem(std::move(input_item), |
+ ShareableBlobDataItem::QUOTA_NEEDED); |
+ pending_transport_items.push_back(item); |
+ transport_items.push_back(item.get()); |
+ checked_transport_quota_needed += length; |
+ } else { |
+ item = new ShareableBlobDataItem( |
+ std::move(input_item), |
+ ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA); |
+ } |
if (length == DataElement::kUnknownSize) |
num_files_with_unknown_size++; |
- scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem( |
- std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA); |
- |
checked_total_size += length; |
output_blob->AppendSharedBlobItem(std::move(item)); |
} |
@@ -206,14 +234,18 @@ BlobStorageContext::BlobFlattener::BlobFlattener( |
return; |
} |
if (!checked_total_size.IsValid() || !checked_total_memory_size.IsValid() || |
- !checked_memory_quota_needed.IsValid()) { |
+ !checked_transport_quota_needed.IsValid() || |
+ !checked_copy_quota_needed.IsValid()) { |
status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS; |
return; |
} |
total_size = checked_total_size.ValueOrDie(); |
total_memory_size = checked_total_memory_size.ValueOrDie(); |
- memory_quota_needed = checked_memory_quota_needed.ValueOrDie(); |
- if (memory_quota_needed) { |
+ transport_quota_needed = checked_transport_quota_needed.ValueOrDie(); |
+ copy_quota_needed = checked_copy_quota_needed.ValueOrDie(); |
+ transport_quota_type = found_file_transport ? TransportQuotaType::FILE |
+ : TransportQuotaType::MEMORY; |
+ if (transport_quota_needed) { |
status = BlobStatus::PENDING_QUOTA; |
} else { |
status = BlobStatus::PENDING_INTERNALS; |
@@ -298,8 +330,19 @@ BlobStorageContext::BlobSlice::BlobSlice(const BlobEntry& source, |
data_item = |
new BlobDataItem(std::move(element), source_item->data_handle_); |
- DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element())) |
- << "File allocation unimplemented."; |
+ if (BlobDataBuilder::IsFutureFileItem(source_item->data_element())) { |
+ // Since we don't have file path / reference for our future file, we |
+ // create another file item with this temporary file name. When our |
+ // blob is finished constructing, all dependent blobs are done, and we |
+ // can copy the handle over. |
+ if (item_index == first_item_index) { |
+ first_item_slice_offset = item_offset; |
+ first_source_item = source_items[item_index]; |
+ } else { |
+ last_source_item = source_items[item_index]; |
+ } |
+ state = ShareableBlobDataItem::QUOTA_NEEDED; |
+ } |
break; |
} |
case DataElement::TYPE_FILE_FILESYSTEM: { |
@@ -341,8 +384,13 @@ BlobStorageContext::BlobStorageContext() |
: memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()), |
ptr_factory_(this) {} |
-BlobStorageContext::~BlobStorageContext() { |
-} |
+BlobStorageContext::BlobStorageContext( |
+ base::FilePath storage_directory, |
+ scoped_refptr<base::TaskRunner> file_runner) |
+ : memory_controller_(std::move(storage_directory), std::move(file_runner)), |
+ ptr_factory_(this) {} |
+ |
+BlobStorageContext::~BlobStorageContext() {} |
std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID( |
const std::string& uuid) { |
@@ -414,14 +462,9 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob( |
// stores the complete item representation in the internal data. |
BlobFlattener flattener(content, entry, ®istry_); |
- DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT || |
- !transport_allowed_callback) |
- << "There is no pending content for the user to populate, so the " |
- "callback should be null."; |
- DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT || |
+ DCHECK(!flattener.contains_unpopulated_transport_items || |
transport_allowed_callback) |
- << "If we have pending content then there needs to be a callback " |
- "present."; |
+ << "If we have pending unpopulated content then a callback is required"; |
entry->set_size(flattener.total_size); |
entry->set_status(flattener.status); |
@@ -430,8 +473,14 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob( |
UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->items().size()); |
UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize", |
flattener.total_memory_size / 1024); |
+ |
+ uint64_t total_memory_needed = |
+ flattener.copy_quota_needed + |
+ (flattener.transport_quota_type == TransportQuotaType::MEMORY |
+ ? flattener.transport_quota_needed |
+ : 0); |
UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize", |
- flattener.memory_quota_needed / 1024); |
+ total_memory_needed / 1024); |
size_t num_building_dependent_blobs = 0; |
std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs; |
@@ -454,7 +503,7 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob( |
} |
entry->set_building_state(base::MakeUnique<BlobEntry::BuildingState>( |
- !flattener.transport_items.empty(), transport_allowed_callback, |
+ !flattener.pending_transport_items.empty(), transport_allowed_callback, |
num_building_dependent_blobs)); |
BlobEntry::BuildingState* building_state = entry->building_state_.get(); |
std::swap(building_state->copies, flattener.copies); |
@@ -468,21 +517,52 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob( |
return handle; |
} |
- if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) { |
+ // Avoid the state where we might grant only one quota. |
+ if (!memory_controller_.CanReserveQuota(flattener.copy_quota_needed + |
+ flattener.transport_quota_needed)) { |
CancelBuildingBlobInternal(entry, BlobStatus::ERR_OUT_OF_MEMORY); |
return handle; |
} |
- if (flattener.memory_quota_needed > 0) { |
+ if (flattener.copy_quota_needed > 0) { |
// The blob can complete during the execution of this line. |
base::WeakPtr<QuotaAllocationTask> pending_request = |
memory_controller_.ReserveMemoryQuota( |
- std::move(flattener.pending_memory_items), |
- base::Bind(&BlobStorageContext::OnEnoughSizeForMemory, |
+ std::move(flattener.pending_copy_items), |
+ base::Bind(&BlobStorageContext::OnEnoughSpaceForCopies, |
ptr_factory_.GetWeakPtr(), content.uuid_)); |
// Building state will be null if the blob is already finished. |
if (entry->building_state_) |
- entry->building_state_->memory_quota_request = std::move(pending_request); |
+ entry->building_state_->copy_quota_request = std::move(pending_request); |
+ } |
+ |
+ if (flattener.transport_quota_needed > 0) { |
+ base::WeakPtr<QuotaAllocationTask> pending_request; |
+ |
+ switch (flattener.transport_quota_type) { |
+ case TransportQuotaType::MEMORY: { |
+ // The blob can complete during the execution of this line. |
+ std::vector<BlobMemoryController::FileCreationInfo> empty_files; |
+ pending_request = memory_controller_.ReserveMemoryQuota( |
+ std::move(flattener.pending_transport_items), |
+ base::Bind(&BlobStorageContext::OnEnoughSpaceForTransport, |
+ ptr_factory_.GetWeakPtr(), content.uuid_, |
+ base::Passed(&empty_files))); |
+ break; |
+ } |
+ case TransportQuotaType::FILE: |
+ pending_request = memory_controller_.ReserveFileQuota( |
+ std::move(flattener.pending_transport_items), |
+ base::Bind(&BlobStorageContext::OnEnoughSpaceForTransport, |
+ ptr_factory_.GetWeakPtr(), content.uuid_)); |
+ break; |
+ } |
+ |
+ // Building state will be null if the blob is already finished. |
+ if (entry->building_state_) { |
+ entry->building_state_->transport_quota_request = |
+ std::move(pending_request); |
+ } |
} |
if (entry->CanFinishBuilding()) |
@@ -622,8 +702,26 @@ void BlobStorageContext::FinishBuilding(BlobEntry* entry) { |
const char* src_data = |
copy.source_item->item()->bytes() + copy.source_item_offset; |
copy.dest_item->item()->item_->SetToBytes(src_data, dest_size); |
- } break; |
- case DataElement::TYPE_FILE: |
+ break; |
+ } |
+ case DataElement::TYPE_FILE: { |
+ // If we expected a memory item (and our source was paged to disk) we |
+ // free that memory. |
+ if (dest_type == DataElement::TYPE_BYTES_DESCRIPTION) |
+ copy.dest_item->set_memory_allocation(nullptr); |
+ |
+ const DataElement& source_element = |
+ copy.source_item->item()->data_element(); |
+ std::unique_ptr<DataElement> new_element(new DataElement()); |
+ new_element->SetToFilePathRange( |
+ source_element.path(), |
+ source_element.offset() + copy.source_item_offset, dest_size, |
+ source_element.expected_modification_time()); |
+ scoped_refptr<BlobDataItem> new_item(new BlobDataItem( |
+ std::move(new_element), copy.source_item->item()->data_handle_)); |
+ copy.dest_item->set_item(std::move(new_item)); |
+ break; |
+ } |
case DataElement::TYPE_UNKNOWN: |
case DataElement::TYPE_BLOB: |
case DataElement::TYPE_BYTES_DESCRIPTION: |
@@ -670,8 +768,10 @@ void BlobStorageContext::RequestTransport( |
NotifyTransportCompleteInternal(entry); |
} |
-void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid, |
- bool success) { |
+void BlobStorageContext::OnEnoughSpaceForTransport( |
+ const std::string& uuid, |
+ std::vector<BlobMemoryController::FileCreationInfo> files, |
+ bool success) { |
if (!success) { |
CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY); |
return; |
@@ -680,15 +780,25 @@ void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid, |
if (!entry || !entry->building_state_.get()) |
return; |
BlobEntry::BuildingState& building_state = *entry->building_state_; |
- DCHECK(!building_state.memory_quota_request); |
+ DCHECK(!building_state.transport_quota_request); |
+ DCHECK(building_state.transport_items_present); |
- if (building_state.transport_items_present) { |
- entry->set_status(BlobStatus::PENDING_TRANSPORT); |
- RequestTransport(entry, |
- std::vector<BlobMemoryController::FileCreationInfo>()); |
- } else { |
- entry->set_status(BlobStatus::PENDING_INTERNALS); |
+ entry->set_status(BlobStatus::PENDING_TRANSPORT); |
+ RequestTransport(entry, std::move(files)); |
+ |
+ if (entry->CanFinishBuilding()) |
+ FinishBuilding(entry); |
+} |
+ |
+void BlobStorageContext::OnEnoughSpaceForCopies(const std::string& uuid, |
+ bool success) { |
+ if (!success) { |
+ CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY); |
+ return; |
} |
+ BlobEntry* entry = registry_.GetEntry(uuid); |
+ if (!entry) |
+ return; |
if (entry->CanFinishBuilding()) |
FinishBuilding(entry); |
@@ -718,8 +828,11 @@ void BlobStorageContext::OnDependentBlobFinished( |
void BlobStorageContext::ClearAndFreeMemory(BlobEntry* entry) { |
if (entry->building_state_) { |
BlobEntry::BuildingState* building_state = entry->building_state_.get(); |
- if (building_state->memory_quota_request) { |
- building_state->memory_quota_request->Cancel(); |
+ if (building_state->copy_quota_request) { |
+ building_state->copy_quota_request->Cancel(); |
+ } |
+ if (building_state->transport_quota_request) { |
+ building_state->transport_quota_request->Cancel(); |
} |
} |
entry->ClearItems(); |