[BlobStorage] Implementing disk.

storage/browser/blob/blob_storage_context.cc

Index: storage/browser/blob/blob_storage_context.cc
diff --git a/storage/browser/blob/blob_storage_context.cc b/storage/browser/blob/blob_storage_context.cc
index 1fecda89f5b6799136cab3bc4089840251ad8f3b..8e2bc51ba91387e9441e40cf63c559109cb802dd 100644
--- a/storage/browser/blob/blob_storage_context.cc
+++ b/storage/browser/blob/blob_storage_context.cc
@@ -92,9 +92,13 @@ BlobStorageContext::BlobFlattener::BlobFlattener(
   size_t num_files_with_unknown_size = 0;
   size_t num_building_dependent_blobs = 0;
 
+  bool found_memory_transport = false;
+  bool found_file_transport = false;
+
   base::CheckedNumeric<uint64_t> checked_total_size = 0;
   base::CheckedNumeric<uint64_t> checked_total_memory_size = 0;
-  base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0;
+  base::CheckedNumeric<uint64_t> checked_transport_quota_needed = 0;
+  base::CheckedNumeric<uint64_t> checked_copy_quota_needed = 0;
 
   for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) {
     const DataElement& input_element = input_item->data_element();
@@ -105,11 +109,19 @@ BlobStorageContext::BlobFlattener::BlobFlattener(
 
     if (IsBytes(type)) {
       DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length());
-      checked_memory_quota_needed += length;
+      found_memory_transport = true;
+      if (found_file_transport) {
+        // We cannot have both memory and file transport items.
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+      contains_unpopulated_transport_items |=
+          (type == DataElement::TYPE_BYTES_DESCRIPTION);
+      checked_transport_quota_needed += length;
       checked_total_size += length;
       scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
           std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED);
-      pending_memory_items.push_back(item);
+      pending_transport_items.push_back(item);
       transport_items.push_back(item.get());
       output_blob->AppendSharedBlobItem(std::move(item));
       continue;
@@ -174,14 +186,14 @@ BlobStorageContext::BlobFlattener::BlobFlattener(
         copies.push_back(ItemCopyEntry(slice.first_source_item,
                                        slice.first_item_slice_offset,
                                        slice.dest_items.front()));
-        pending_memory_items.push_back(slice.dest_items.front());
+        pending_copy_items.push_back(slice.dest_items.front());
       }
       if (slice.last_source_item) {
         copies.push_back(
             ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back()));
-        pending_memory_items.push_back(slice.dest_items.back());
+        pending_copy_items.push_back(slice.dest_items.back());
       }
-      checked_memory_quota_needed += slice.copying_memory_size;
+      checked_copy_quota_needed += slice.copying_memory_size;
 
       for (auto& shareable_item : slice.dest_items) {
         output_blob->AppendSharedBlobItem(std::move(shareable_item));
@@ -189,14 +201,30 @@ BlobStorageContext::BlobFlattener::BlobFlattener(
       continue;
     }
 
-    DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element))
-        << "File allocation not implemented.";
+    // If the source item is a temporary file item, then we need to keep track
+    // of that and mark it as needing quota.
+    scoped_refptr<ShareableBlobDataItem> item;
+    if (BlobDataBuilder::IsFutureFileItem(input_element)) {
+      found_file_transport = true;
+      if (found_memory_transport) {
+        // We cannot have both memory and file transport items.
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+      contains_unpopulated_transport_items = true;
+      item = new ShareableBlobDataItem(std::move(input_item),
+                                       ShareableBlobDataItem::QUOTA_NEEDED);
+      pending_transport_items.push_back(item);
+      transport_items.push_back(item.get());
+      checked_transport_quota_needed += length;
+    } else {
+      item = new ShareableBlobDataItem(
+          std::move(input_item),
+          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
+    }
     if (length == DataElement::kUnknownSize)
       num_files_with_unknown_size++;
 
-    scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
-        std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
-
     checked_total_size += length;
     output_blob->AppendSharedBlobItem(std::move(item));
   }
@@ -206,14 +234,18 @@ BlobStorageContext::BlobFlattener::BlobFlattener(
     return;
   }
   if (!checked_total_size.IsValid() || !checked_total_memory_size.IsValid() ||
-      !checked_memory_quota_needed.IsValid()) {
+      !checked_transport_quota_needed.IsValid() ||
+      !checked_copy_quota_needed.IsValid()) {
     status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
     return;
   }
   total_size = checked_total_size.ValueOrDie();
   total_memory_size = checked_total_memory_size.ValueOrDie();
-  memory_quota_needed = checked_memory_quota_needed.ValueOrDie();
-  if (memory_quota_needed) {
+  transport_quota_needed = checked_transport_quota_needed.ValueOrDie();
+  copy_quota_needed = checked_copy_quota_needed.ValueOrDie();
+  transport_quota_type = found_file_transport ? TransportQuotaType::FILE
+                                              : TransportQuotaType::MEMORY;
+  if (transport_quota_needed) {
     status = BlobStatus::PENDING_QUOTA;
   } else {
     status = BlobStatus::PENDING_INTERNALS;
@@ -298,8 +330,19 @@ BlobStorageContext::BlobSlice::BlobSlice(const BlobEntry& source,
         data_item =
             new BlobDataItem(std::move(element), source_item->data_handle_);
 
-        DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element()))
-            << "File allocation unimplemented.";
+        if (BlobDataBuilder::IsFutureFileItem(source_item->data_element())) {
+          // Since we don't have file path / reference for our future file, we
+          // create another file item with this temporary file name. When our
+          // blob is finished constructing, all dependent blobs are done, and we
+          // can copy the handle over.
+          if (item_index == first_item_index) {
+            first_item_slice_offset = item_offset;
+            first_source_item = source_items[item_index];
+          } else {
+            last_source_item = source_items[item_index];
+          }
+          state = ShareableBlobDataItem::QUOTA_NEEDED;
+        }
         break;
       }
       case DataElement::TYPE_FILE_FILESYSTEM: {
@@ -341,8 +384,13 @@ BlobStorageContext::BlobStorageContext()
     : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()),
       ptr_factory_(this) {}
 
-BlobStorageContext::~BlobStorageContext() {
-}
+BlobStorageContext::BlobStorageContext(
+    base::FilePath storage_directory,
+    scoped_refptr<base::TaskRunner> file_runner)
+    : memory_controller_(std::move(storage_directory), std::move(file_runner)),
+      ptr_factory_(this) {}
+
+BlobStorageContext::~BlobStorageContext() {}
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID(
     const std::string& uuid) {
@@ -414,14 +462,9 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
   // stores the complete item representation in the internal data.
   BlobFlattener flattener(content, entry, &registry_);
 
-  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
-         !transport_allowed_callback)
-      << "There is no pending content for the user to populate, so the "
-         "callback should be null.";
-  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+  DCHECK(!flattener.contains_unpopulated_transport_items ||
          transport_allowed_callback)
-      << "If we have pending content then there needs to be a callback "
-         "present.";
+      << "If we have pending unpopulated content then a callback is required";
 
   entry->set_size(flattener.total_size);
   entry->set_status(flattener.status);
@@ -430,8 +473,14 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
   UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->items().size());
   UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize",
                        flattener.total_memory_size / 1024);
+
+  uint64_t total_memory_needed =
+      flattener.copy_quota_needed +
+      (flattener.transport_quota_type == TransportQuotaType::MEMORY
+           ? flattener.transport_quota_needed
+           : 0);
   UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
-                       flattener.memory_quota_needed / 1024);
+                       total_memory_needed / 1024);
 
   size_t num_building_dependent_blobs = 0;
   std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs;
@@ -454,7 +503,7 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
   }
 
   entry->set_building_state(base::MakeUnique<BlobEntry::BuildingState>(
-      !flattener.transport_items.empty(), transport_allowed_callback,
+      !flattener.pending_transport_items.empty(), transport_allowed_callback,
       num_building_dependent_blobs));
   BlobEntry::BuildingState* building_state = entry->building_state_.get();
   std::swap(building_state->copies, flattener.copies);
@@ -468,21 +517,52 @@ std::unique_ptr<BlobDataHandle> BlobStorageContext::BuildBlob(
     return handle;
   }
 
-  if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) {
+  // Avoid the state where we might grant only one quota.
+  if (!memory_controller_.CanReserveQuota(flattener.copy_quota_needed +
+                                          flattener.transport_quota_needed)) {
     CancelBuildingBlobInternal(entry, BlobStatus::ERR_OUT_OF_MEMORY);
     return handle;
   }
 
-  if (flattener.memory_quota_needed > 0) {
+  if (flattener.copy_quota_needed > 0) {
     // The blob can complete during the execution of this line.
     base::WeakPtr<QuotaAllocationTask> pending_request =
         memory_controller_.ReserveMemoryQuota(
-            std::move(flattener.pending_memory_items),
-            base::Bind(&BlobStorageContext::OnEnoughSizeForMemory,
+            std::move(flattener.pending_copy_items),
+            base::Bind(&BlobStorageContext::OnEnoughSpaceForCopies,
                        ptr_factory_.GetWeakPtr(), content.uuid_));
     // Building state will be null if the blob is already finished.
     if (entry->building_state_)
-      entry->building_state_->memory_quota_request = std::move(pending_request);
+      entry->building_state_->copy_quota_request = std::move(pending_request);
+  }
+
+  if (flattener.transport_quota_needed > 0) {
+    base::WeakPtr<QuotaAllocationTask> pending_request;
+
+    switch (flattener.transport_quota_type) {
+      case TransportQuotaType::MEMORY: {
+        // The blob can complete during the execution of this line.
+        std::vector<BlobMemoryController::FileCreationInfo> empty_files;
+        pending_request = memory_controller_.ReserveMemoryQuota(
+            std::move(flattener.pending_transport_items),
+            base::Bind(&BlobStorageContext::OnEnoughSpaceForTransport,
+                       ptr_factory_.GetWeakPtr(), content.uuid_,
+                       base::Passed(&empty_files)));
+        break;
+      }
+      case TransportQuotaType::FILE:
+        pending_request = memory_controller_.ReserveFileQuota(
+            std::move(flattener.pending_transport_items),
+            base::Bind(&BlobStorageContext::OnEnoughSpaceForTransport,
+                       ptr_factory_.GetWeakPtr(), content.uuid_));
+        break;
+    }
+
+    // Building state will be null if the blob is already finished.
+    if (entry->building_state_) {
+      entry->building_state_->transport_quota_request =
+          std::move(pending_request);
+    }
   }
 
   if (entry->CanFinishBuilding())
@@ -605,6 +685,7 @@ void BlobStorageContext::FinishBuilding(BlobEntry* entry) {
   bool error = BlobStatusIsError(status);
   UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken", error);
   if (error) {
+    LOG(ERROR) << "Blob is broken, reason: " << static_cast<int>(status);

[michaeln, 2016/12/02 20:41:08]

In general, please avoid using LOG except for very consequential events, prefer
DLOG or VLOG.

This particular event is histogram logged and appears in chrome://histograms in
dbg and rel builds. I think I'd vote to not add it to the console output too.

[dmurph, 2016/12/02 21:53:07]

Done, this was to figure out the browsertest error on windows.

     UMA_HISTOGRAM_ENUMERATION("Storage.Blob.BrokenReason",
                               static_cast<int>(status),
                               (static_cast<int>(BlobStatus::LAST_ERROR) + 1));
@@ -622,8 +703,26 @@ void BlobStorageContext::FinishBuilding(BlobEntry* entry) {
           const char* src_data =
               copy.source_item->item()->bytes() + copy.source_item_offset;
           copy.dest_item->item()->item_->SetToBytes(src_data, dest_size);
-        } break;
-        case DataElement::TYPE_FILE:
+          break;
+        }
+        case DataElement::TYPE_FILE: {
+          // If we expected a memory item (and our source was paged to disk) we
+          // free that memory.
+          if (dest_type == DataElement::TYPE_BYTES_DESCRIPTION)
+            copy.dest_item->set_memory_allocation(nullptr);
+
+          const DataElement& source_element =
+              copy.source_item->item()->data_element();
+          std::unique_ptr<DataElement> new_element(new DataElement());
+          new_element->SetToFilePathRange(
+              source_element.path(),
+              source_element.offset() + copy.source_item_offset, dest_size,
+              source_element.expected_modification_time());
+          scoped_refptr<BlobDataItem> new_item(new BlobDataItem(
+              std::move(new_element), copy.source_item->item()->data_handle_));
+          copy.dest_item->set_item(std::move(new_item));
+          break;
+        }
         case DataElement::TYPE_UNKNOWN:
         case DataElement::TYPE_BLOB:
         case DataElement::TYPE_BYTES_DESCRIPTION:
@@ -670,8 +769,10 @@ void BlobStorageContext::RequestTransport(
   NotifyTransportCompleteInternal(entry);
 }
 
-void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid,
-                                               bool success) {
+void BlobStorageContext::OnEnoughSpaceForTransport(
+    const std::string& uuid,
+    std::vector<BlobMemoryController::FileCreationInfo> files,
+    bool success) {
   if (!success) {
     CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
     return;
@@ -680,15 +781,25 @@ void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid,
   if (!entry || !entry->building_state_.get())
     return;
   BlobEntry::BuildingState& building_state = *entry->building_state_;
-  DCHECK(!building_state.memory_quota_request);
+  DCHECK(!building_state.transport_quota_request);
+  DCHECK(building_state.transport_items_present);
 
-  if (building_state.transport_items_present) {
-    entry->set_status(BlobStatus::PENDING_TRANSPORT);
-    RequestTransport(entry,
-                     std::vector<BlobMemoryController::FileCreationInfo>());
-  } else {
-    entry->set_status(BlobStatus::PENDING_INTERNALS);
+  entry->set_status(BlobStatus::PENDING_TRANSPORT);
+  RequestTransport(entry, std::move(files));
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
+}
+
+void BlobStorageContext::OnEnoughSpaceForCopies(const std::string& uuid,
+                                                bool success) {
+  if (!success) {
+    CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+    return;
   }
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  if (!entry)
+    return;
 
   if (entry->CanFinishBuilding())
     FinishBuilding(entry);
@@ -716,12 +827,8 @@ void BlobStorageContext::OnDependentBlobFinished(
 }
 
 void BlobStorageContext::ClearAndFreeMemory(BlobEntry* entry) {
-  if (entry->building_state_) {
-    BlobEntry::BuildingState* building_state = entry->building_state_.get();
-    if (building_state->memory_quota_request) {
-      building_state->memory_quota_request->Cancel();
-    }
-  }
+  if (entry->building_state_)
+    entry->building_state_->CancelRequests();
   entry->ClearItems();
   entry->ClearOffsets();
   entry->set_size(0);