[BlobStorage] Implementing disk.

storage/browser/blob/blob_storage_context.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_storage_context.h"
 
 #include <algorithm>
 #include <limits>
 #include <memory>
 #include <set>
@@ skipping 74 matching lines @@
 BlobStorageContext::BlobFlattener::BlobFlattener(
     const BlobDataBuilder& input_builder,
     BlobEntry* output_blob,
     BlobStorageRegistry* registry) {
   const std::string& uuid = input_builder.uuid_;
   std::set<std::string> dependent_blob_uuids;
 
   size_t num_files_with_unknown_size = 0;
   size_t num_building_dependent_blobs = 0;
 
+  bool found_memory_transport = false;
+  bool found_file_transport = false;
+
   base::CheckedNumeric<uint64_t> checked_total_size = 0;
   base::CheckedNumeric<uint64_t> checked_total_memory_size = 0;
-  base::CheckedNumeric<uint64_t> checked_memory_quota_needed = 0;
+  base::CheckedNumeric<uint64_t> checked_transport_quota_needed = 0;
+  base::CheckedNumeric<uint64_t> checked_copy_quota_needed = 0;
 
   for (scoped_refptr<BlobDataItem> input_item : input_builder.items_) {
     const DataElement& input_element = input_item->data_element();
     DataElement::Type type = input_element.type();
     uint64_t length = input_element.length();
 
     RecordBlobItemSizeStats(input_element);
 
     if (IsBytes(type)) {
       DCHECK_NE(0 + DataElement::kUnknownSize, input_element.length());
-      checked_memory_quota_needed += length;
+      found_memory_transport = true;
+      if (found_file_transport) {
+        // We cannot have both memory and file transport items.
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+      contains_unpopulated_transport_items |=
+          (type == DataElement::TYPE_BYTES_DESCRIPTION);
+      checked_transport_quota_needed += length;
       checked_total_size += length;
       scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
           std::move(input_item), ShareableBlobDataItem::QUOTA_NEEDED);
-      pending_memory_items.push_back(item);
+      pending_transport_items.push_back(item);
       transport_items.push_back(item.get());
       output_blob->AppendSharedBlobItem(std::move(item));
       continue;
     }
     if (type == DataElement::TYPE_BLOB) {
       BlobEntry* ref_entry = registry->GetEntry(input_element.blob_uuid());
 
       if (!ref_entry || input_element.blob_uuid() == uuid) {
         status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
         return;
@@ skipping 44 matching lines @@
           !slice.total_memory_size.IsValid()) {
         status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
         return;
       }
       checked_total_memory_size += slice.total_memory_size;
 
       if (slice.first_source_item) {
         copies.push_back(ItemCopyEntry(slice.first_source_item,
                                        slice.first_item_slice_offset,
                                        slice.dest_items.front()));
-        pending_memory_items.push_back(slice.dest_items.front());
+        pending_copy_items.push_back(slice.dest_items.front());
       }
       if (slice.last_source_item) {
         copies.push_back(
             ItemCopyEntry(slice.last_source_item, 0, slice.dest_items.back()));
-        pending_memory_items.push_back(slice.dest_items.back());
+        pending_copy_items.push_back(slice.dest_items.back());
       }
-      checked_memory_quota_needed += slice.copying_memory_size;
+      checked_copy_quota_needed += slice.copying_memory_size;
 
       for (auto& shareable_item : slice.dest_items) {
         output_blob->AppendSharedBlobItem(std::move(shareable_item));
       }
       continue;
     }
 
-    DCHECK(!BlobDataBuilder::IsFutureFileItem(input_element))
-        << "File allocation not implemented.";
+    // If the source item is a temporary file item, then we need to keep track
+    // of that and mark it as needing quota.
+    scoped_refptr<ShareableBlobDataItem> item;
+    if (BlobDataBuilder::IsFutureFileItem(input_element)) {
+      found_file_transport = true;
+      if (found_memory_transport) {
+        // We cannot have both memory and file transport items.
+        status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
+        return;
+      }
+      contains_unpopulated_transport_items = true;
+      item = new ShareableBlobDataItem(std::move(input_item),
+                                       ShareableBlobDataItem::QUOTA_NEEDED);
+      pending_transport_items.push_back(item);
+      transport_items.push_back(item.get());
+      checked_transport_quota_needed += length;
+    } else {
+      item = new ShareableBlobDataItem(
+          std::move(input_item),
+          ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
+    }
     if (length == DataElement::kUnknownSize)
       num_files_with_unknown_size++;
 
-    scoped_refptr<ShareableBlobDataItem> item = new ShareableBlobDataItem(
-        std::move(input_item), ShareableBlobDataItem::POPULATED_WITHOUT_QUOTA);
-
     checked_total_size += length;
     output_blob->AppendSharedBlobItem(std::move(item));
   }
 
   if (num_files_with_unknown_size > 1 && input_builder.items_.size() > 1) {
     status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
     return;
   }
   if (!checked_total_size.IsValid() || !checked_total_memory_size.IsValid() ||
-      !checked_memory_quota_needed.IsValid()) {
+      !checked_transport_quota_needed.IsValid() ||
+      !checked_copy_quota_needed.IsValid()) {
     status = BlobStatus::ERR_INVALID_CONSTRUCTION_ARGUMENTS;
     return;
   }
   total_size = checked_total_size.ValueOrDie();
   total_memory_size = checked_total_memory_size.ValueOrDie();
-  memory_quota_needed = checked_memory_quota_needed.ValueOrDie();
-  if (memory_quota_needed) {
+  transport_quota_needed = checked_transport_quota_needed.ValueOrDie();
+  copy_quota_needed = checked_copy_quota_needed.ValueOrDie();
+  transport_quota_type = found_file_transport ? TransportQuotaType::FILE
+                                              : TransportQuotaType::MEMORY;
+  if (transport_quota_needed) {
     status = BlobStatus::PENDING_QUOTA;
   } else {
     status = BlobStatus::PENDING_INTERNALS;
   }
 }
 
 BlobStorageContext::BlobFlattener::~BlobFlattener() {}
 
 BlobStorageContext::BlobSlice::BlobSlice(const BlobEntry& source,
                                          uint64_t slice_offset,
@@ skipping 64 matching lines @@
       case DataElement::TYPE_FILE: {
         UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.File",
                              read_size / 1024);
         std::unique_ptr<DataElement> element(new DataElement());
         element->SetToFilePathRange(
             source_item->path(), source_item->offset() + item_offset, read_size,
             source_item->expected_modification_time());
         data_item =
             new BlobDataItem(std::move(element), source_item->data_handle_);
 
-        DCHECK(!BlobDataBuilder::IsFutureFileItem(source_item->data_element()))
-            << "File allocation unimplemented.";
+        if (BlobDataBuilder::IsFutureFileItem(source_item->data_element())) {
+          // Since we don't have file path / reference for our future file, we
+          // create another file item with this temporary file name. When our
+          // blob is finished constructing, all dependent blobs are done, and we
+          // can copy the handle over.
+          if (item_index == first_item_index) {
+            first_item_slice_offset = item_offset;
+            first_source_item = source_items[item_index];
+          } else {
+            last_source_item = source_items[item_index];
+          }
+          state = ShareableBlobDataItem::QUOTA_NEEDED;
+        }
         break;
       }
       case DataElement::TYPE_FILE_FILESYSTEM: {
         UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
                              read_size / 1024);
         std::unique_ptr<DataElement> element(new DataElement());
         element->SetToFileSystemUrlRange(
             source_item->filesystem_url(), source_item->offset() + item_offset,
             read_size, source_item->expected_modification_time());
         data_item = new BlobDataItem(std::move(element));
@@ skipping 21 matching lines @@
     item_offset = 0;
   }
 }
 
 BlobStorageContext::BlobSlice::~BlobSlice() {}
 
 BlobStorageContext::BlobStorageContext()
     : memory_controller_(base::FilePath(), scoped_refptr<base::TaskRunner>()),
       ptr_factory_(this) {}
 
-BlobStorageContext::~BlobStorageContext() {
-}
+BlobStorageContext::BlobStorageContext(
+    base::FilePath storage_directory,
+    scoped_refptr<base::TaskRunner> file_runner)
+    : memory_controller_(std::move(storage_directory), std::move(file_runner)),
+      ptr_factory_(this) {}
+
+BlobStorageContext::~BlobStorageContext() {}
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromUUID(
     const std::string& uuid) {
   BlobEntry* entry = registry_.GetEntry(uuid);
   if (!entry)
     return nullptr;
   return CreateHandle(uuid, entry);
 }
 
 std::unique_ptr<BlobDataHandle> BlobStorageContext::GetBlobDataFromPublicURL(
@@ skipping 51 matching lines @@
     const TransportAllowedCallback& transport_allowed_callback) {
   DCHECK(!registry_.HasEntry(content.uuid_));
 
   BlobEntry* entry = registry_.CreateEntry(
       content.uuid(), content.content_type_, content.content_disposition_);
 
   // This flattens all blob references in the transportion content out and
   // stores the complete item representation in the internal data.
   BlobFlattener flattener(content, entry, &registry_);
 
-  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
-         !transport_allowed_callback)
-      << "There is no pending content for the user to populate, so the "
-         "callback should be null.";
-  DCHECK(flattener.status != BlobStatus::PENDING_TRANSPORT ||
+  DCHECK(!flattener.contains_unpopulated_transport_items ||
          transport_allowed_callback)
-      << "If we have pending content then there needs to be a callback "
-         "present.";
+      << "If we have pending unpopulated content then a callback is required";
 
   entry->set_size(flattener.total_size);
   entry->set_status(flattener.status);
   std::unique_ptr<BlobDataHandle> handle = CreateHandle(content.uuid_, entry);
 
   UMA_HISTOGRAM_COUNTS("Storage.Blob.ItemCount", entry->items().size());
   UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalSize",
                        flattener.total_memory_size / 1024);
+
+  uint64_t total_memory_needed =
+      flattener.copy_quota_needed +
+      (flattener.transport_quota_type == TransportQuotaType::MEMORY
+           ? flattener.transport_quota_needed
+           : 0);
   UMA_HISTOGRAM_COUNTS("Storage.Blob.TotalUnsharedSize",
-                       flattener.memory_quota_needed / 1024);
+                       total_memory_needed / 1024);
 
   size_t num_building_dependent_blobs = 0;
   std::vector<std::unique_ptr<BlobDataHandle>> dependent_blobs;
   // We hold a handle to all blobs we're using. This is important, as our memory
   // accounting can be delayed until OnEnoughSizeForBlobData is called, and we
   // only free memory on canceling when we've done this accounting. If a
   // dependent blob is dereferenced, then we're the last blob holding onto that
   // data item, and we need to account for that. So we prevent that case by
   // holding onto all blobs.
   for (const std::pair<std::string, BlobEntry*>& pending_blob :
        flattener.dependent_blobs) {
     dependent_blobs.push_back(
         CreateHandle(pending_blob.first, pending_blob.second));
     if (BlobStatusIsPending(pending_blob.second->status())) {
       pending_blob.second->building_state_->build_completion_callbacks
           .push_back(base::Bind(&BlobStorageContext::OnDependentBlobFinished,
                                 ptr_factory_.GetWeakPtr(), content.uuid_));
       num_building_dependent_blobs++;
     }
   }
 
   entry->set_building_state(base::MakeUnique<BlobEntry::BuildingState>(
-      !flattener.transport_items.empty(), transport_allowed_callback,
+      !flattener.pending_transport_items.empty(), transport_allowed_callback,
       num_building_dependent_blobs));
   BlobEntry::BuildingState* building_state = entry->building_state_.get();
   std::swap(building_state->copies, flattener.copies);
   std::swap(building_state->dependent_blobs, dependent_blobs);
   std::swap(building_state->transport_items, flattener.transport_items);
 
   // Break ourselves if we have an error. BuildingState must be set first so the
   // callback is called correctly.
   if (BlobStatusIsError(flattener.status)) {
     CancelBuildingBlobInternal(entry, flattener.status);
     return handle;
   }
 
-  if (!memory_controller_.CanReserveQuota(flattener.memory_quota_needed)) {
+  // Avoid the state where we might grant only one quota.
+  if (!memory_controller_.CanReserveQuota(flattener.copy_quota_needed +
+                                          flattener.transport_quota_needed)) {
     CancelBuildingBlobInternal(entry, BlobStatus::ERR_OUT_OF_MEMORY);
     return handle;
   }
 
-  if (flattener.memory_quota_needed > 0) {
+  if (flattener.copy_quota_needed > 0) {
     // The blob can complete during the execution of this line.
     base::WeakPtr<QuotaAllocationTask> pending_request =
         memory_controller_.ReserveMemoryQuota(
-            std::move(flattener.pending_memory_items),
-            base::Bind(&BlobStorageContext::OnEnoughSizeForMemory,
+            std::move(flattener.pending_copy_items),
+            base::Bind(&BlobStorageContext::OnEnoughSpaceForCopies,
                        ptr_factory_.GetWeakPtr(), content.uuid_));
     // Building state will be null if the blob is already finished.
     if (entry->building_state_)
-      entry->building_state_->memory_quota_request = std::move(pending_request);
+      entry->building_state_->copy_quota_request = std::move(pending_request);
+  }
+
+  if (flattener.transport_quota_needed > 0) {
+    base::WeakPtr<QuotaAllocationTask> pending_request;
+
+    switch (flattener.transport_quota_type) {
+      case TransportQuotaType::MEMORY: {
+        // The blob can complete during the execution of this line.
+        std::vector<BlobMemoryController::FileCreationInfo> empty_files;
+        pending_request = memory_controller_.ReserveMemoryQuota(
+            std::move(flattener.pending_transport_items),
+            base::Bind(&BlobStorageContext::OnEnoughSpaceForTransport,
+                       ptr_factory_.GetWeakPtr(), content.uuid_,
+                       base::Passed(&empty_files)));
+        break;
+      }
+      case TransportQuotaType::FILE:
+        pending_request = memory_controller_.ReserveFileQuota(
+            std::move(flattener.pending_transport_items),
+            base::Bind(&BlobStorageContext::OnEnoughSpaceForTransport,
+                       ptr_factory_.GetWeakPtr(), content.uuid_));
+        break;
+    }
+
+    // Building state will be null if the blob is already finished.
+    if (entry->building_state_) {
+      entry->building_state_->transport_quota_request =
+          std::move(pending_request);
+    }
   }
 
   if (entry->CanFinishBuilding())
     FinishBuilding(entry);
 
   return handle;
 }
 
 void BlobStorageContext::CancelBuildingBlob(const std::string& uuid,
                                             BlobStatus reason) {
@@ skipping 102 matching lines @@
 
 void BlobStorageContext::FinishBuilding(BlobEntry* entry) {
   DCHECK(entry);
 
   BlobStatus status = entry->status_;
   DCHECK_NE(BlobStatus::DONE, status);
 
   bool error = BlobStatusIsError(status);
   UMA_HISTOGRAM_BOOLEAN("Storage.Blob.Broken", error);
   if (error) {
+    LOG(ERROR) << "Blob is broken, reason: " << static_cast<int>(status);

[michaeln, 2016/12/02 20:41:08]

In general, please avoid using LOG except for very consequential events, prefer
DLOG or VLOG.

This particular event is histogram logged and appears in chrome://histograms in
dbg and rel builds. I think I'd vote to not add it to the console output too.

[dmurph, 2016/12/02 21:53:07]

Done, this was to figure out the browsertest error on windows.

     UMA_HISTOGRAM_ENUMERATION("Storage.Blob.BrokenReason",
                               static_cast<int>(status),
                               (static_cast<int>(BlobStatus::LAST_ERROR) + 1));
   }
 
   if (BlobStatusIsPending(entry->status_)) {
     for (const ItemCopyEntry& copy : entry->building_state_->copies) {
       // Our source item can be a file if it was a slice of an unpopulated file,
       // or a slice of data that was then paged to disk.
       size_t dest_size = static_cast<size_t>(copy.dest_item->item()->length());
       DataElement::Type dest_type = copy.dest_item->item()->type();
       switch (copy.source_item->item()->type()) {
         case DataElement::TYPE_BYTES: {
           DCHECK_EQ(dest_type, DataElement::TYPE_BYTES_DESCRIPTION);
           const char* src_data =
               copy.source_item->item()->bytes() + copy.source_item_offset;
           copy.dest_item->item()->item_->SetToBytes(src_data, dest_size);
-        } break;
-        case DataElement::TYPE_FILE:
+          break;
+        }
+        case DataElement::TYPE_FILE: {
+          // If we expected a memory item (and our source was paged to disk) we
+          // free that memory.
+          if (dest_type == DataElement::TYPE_BYTES_DESCRIPTION)
+            copy.dest_item->set_memory_allocation(nullptr);
+
+          const DataElement& source_element =
+              copy.source_item->item()->data_element();
+          std::unique_ptr<DataElement> new_element(new DataElement());
+          new_element->SetToFilePathRange(
+              source_element.path(),
+              source_element.offset() + copy.source_item_offset, dest_size,
+              source_element.expected_modification_time());
+          scoped_refptr<BlobDataItem> new_item(new BlobDataItem(
+              std::move(new_element), copy.source_item->item()->data_handle_));
+          copy.dest_item->set_item(std::move(new_item));
+          break;
+        }
         case DataElement::TYPE_UNKNOWN:
         case DataElement::TYPE_BLOB:
         case DataElement::TYPE_BYTES_DESCRIPTION:
         case DataElement::TYPE_FILE_FILESYSTEM:
         case DataElement::TYPE_DISK_CACHE_ENTRY:
           NOTREACHED();
           break;
       }
       copy.dest_item->set_state(ShareableBlobDataItem::POPULATED_WITH_QUOTA);
     }
@@ skipping 26 matching lines @@
   BlobEntry::BuildingState* building_state = entry->building_state_.get();
   if (building_state->transport_allowed_callback) {
     base::ResetAndReturn(&building_state->transport_allowed_callback)
         .Run(BlobStatus::PENDING_TRANSPORT, std::move(files));
     return;
   }
   DCHECK(files.empty());
   NotifyTransportCompleteInternal(entry);
 }
 
-void BlobStorageContext::OnEnoughSizeForMemory(const std::string& uuid,
-                                               bool success) {
+void BlobStorageContext::OnEnoughSpaceForTransport(
+    const std::string& uuid,
+    std::vector<BlobMemoryController::FileCreationInfo> files,
+    bool success) {
   if (!success) {
     CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
     return;
   }
   BlobEntry* entry = registry_.GetEntry(uuid);
   if (!entry || !entry->building_state_.get())
     return;
   BlobEntry::BuildingState& building_state = *entry->building_state_;
-  DCHECK(!building_state.memory_quota_request);
+  DCHECK(!building_state.transport_quota_request);
+  DCHECK(building_state.transport_items_present);
 
-  if (building_state.transport_items_present) {
-    entry->set_status(BlobStatus::PENDING_TRANSPORT);
-    RequestTransport(entry,
-                     std::vector<BlobMemoryController::FileCreationInfo>());
-  } else {
-    entry->set_status(BlobStatus::PENDING_INTERNALS);
-  }
+  entry->set_status(BlobStatus::PENDING_TRANSPORT);
+  RequestTransport(entry, std::move(files));
 
   if (entry->CanFinishBuilding())
     FinishBuilding(entry);
+}
+
+void BlobStorageContext::OnEnoughSpaceForCopies(const std::string& uuid,
+                                                bool success) {
+  if (!success) {
+    CancelBuildingBlob(uuid, BlobStatus::ERR_OUT_OF_MEMORY);
+    return;
+  }
+  BlobEntry* entry = registry_.GetEntry(uuid);
+  if (!entry)
+    return;
+
+  if (entry->CanFinishBuilding())
+    FinishBuilding(entry);
 }
 
 void BlobStorageContext::OnDependentBlobFinished(
     const std::string& owning_blob_uuid,
     BlobStatus status) {
   BlobEntry* entry = registry_.GetEntry(owning_blob_uuid);
   if (!entry || !entry->building_state_)
     return;
 
   if (BlobStatusIsError(status)) {
     DCHECK_NE(BlobStatus::ERR_BLOB_DEREFERENCED_WHILE_BUILDING, status)
         << "Referenced blob should never be dereferenced while we "
         << "are depending on it, as our system holds a handle.";
     CancelBuildingBlobInternal(entry, BlobStatus::ERR_REFERENCED_BLOB_BROKEN);
     return;
   }
   DCHECK_GT(entry->building_state_->num_building_dependent_blobs, 0u);
   --entry->building_state_->num_building_dependent_blobs;
 
   if (entry->CanFinishBuilding())
     FinishBuilding(entry);
 }
 
 void BlobStorageContext::ClearAndFreeMemory(BlobEntry* entry) {
-  if (entry->building_state_) {
-    BlobEntry::BuildingState* building_state = entry->building_state_.get();
-    if (building_state->memory_quota_request) {
-      building_state->memory_quota_request->Cancel();
-    }
-  }
+  if (entry->building_state_)
+    entry->building_state_->CancelRequests();
   entry->ClearItems();
   entry->ClearOffsets();
   entry->set_size(0);
 }
 
 }  // namespace storage