[Blob] Blob paging to disk patch.

storage/browser/blob/blob_memory_controller.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "storage/browser/blob/blob_memory_controller.h"
+
+#include <algorithm>
+
+#include "base/callback.h"
+#include "base/callback_helpers.h"
+#include "base/files/file_util.h"
+#include "base/location.h"
+#include "base/metrics/histogram_macros.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "base/single_thread_task_runner.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/task_runner.h"
+#include "base/task_runner_util.h"
+#include "base/thread_task_runner_handle.h"
+#include "base/time/time.h"
+#include "base/trace_event/trace_event.h"
+#include "base/tuple.h"
+#include "storage/browser/blob/blob_data_item.h"
+#include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/browser/blob/shareable_file_reference.h"
+
+using base::File;
+using base::FilePath;
+
+namespace storage {
+
+BlobMemoryController::FileCreationInfo::FileCreationInfo() {}
+BlobMemoryController::FileCreationInfo::~FileCreationInfo() {}
+
+BlobMemoryController::BlobMemoryController()
+    : recent_item_cache_(RecentItemsCache::NO_AUTO_EVICT) {}
+
+BlobMemoryController::BlobMemoryController(
+    bool enable_disk,
+    const FilePath& blob_storage_dir,
+    scoped_refptr<base::TaskRunner> file_runner)
+    : file_runner_(file_runner),
+      enable_disk_(enable_disk),
+      blob_storage_dir_(blob_storage_dir),
+      recent_item_cache_(RecentItemsCache::NO_AUTO_EVICT) {}
+
+BlobMemoryController::~BlobMemoryController() {}
+
+bool BlobMemoryController::DecideBlobTransportationMemoryStrategy(
+    const std::vector<DataElement>& descriptions,
+    uint64_t* total_bytes,
+    BlobMemoryController::BlobTrasportationMemoryStrategyResult* result) {
+  DCHECK(total_bytes);
+  DCHECK(result);
+  // Step 1: Get the sizes.
+  size_t shortcut_memory_size_bytes;
+  uint64_t total_memory_size_bytes;
+  if (!CalculateBlobMemorySize(descriptions, &shortcut_memory_size_bytes,
+                               &total_memory_size_bytes)) {
+    return false;
+  }
+  *total_bytes = total_memory_size_bytes;
+
+  // Step 2: Handle case where we have no memory to transport.
+  if (total_memory_size_bytes == 0) {
+    *result = BlobTrasportationMemoryStrategyResult::NONE_NEEDED;
+    return true;
+  }
+
+  // Step 3: Check if we have enough memory to store the blob.
+  if (total_memory_size_bytes > GetAvailableMemoryForBlobs() &&
+      total_memory_size_bytes > GetAvailableDiskSpaceForBlobs()) {
+    *result = BlobTrasportationMemoryStrategyResult::TOO_LARGE;
+    return true;
+  }
+  // From here on, we know we can fit the blob in memory or on disk.
+
+  // Step 4: Decide if we're using the shortcut method.
+  if (shortcut_memory_size_bytes == total_memory_size_bytes &&
+      blobs_waiting_for_paging_.empty() &&
+      shortcut_memory_size_bytes < GetAvailableMemoryForBlobs()) {
+    *result = BlobTrasportationMemoryStrategyResult::SHORTCUT;
+    return true;
+  }
+
+  // Step 5: Decide if we're going straight to disk.
+  if (enable_disk_ && (total_memory_size_bytes > max_blob_in_memory_size_)) {
+    *result = BlobTrasportationMemoryStrategyResult::FILE;
+    return true;
+  }
+  // From here on, we know the blob's size is less than:
+  // * max_blob_in_memory_size_ if enable_disk_ is true
+  // * max_blob_memory_space_ if enable_disk_ is false
+  // So we know we're < max(size_t).
+
+  // Step 6: Decide if we're using shared memory.
+  if (total_memory_size_bytes > max_ipc_memory_size_) {
+    *result = BlobTrasportationMemoryStrategyResult::SHARED_MEMORY;
+    return true;
+  }
+
+  // Step 7: We can fit in IPC.
+  *result = BlobTrasportationMemoryStrategyResult::IPC;
+  return true;
+}
+
+void BlobMemoryController::CreateTemporaryFileForRenderer(
+    uint64_t size_bytes,
+    base::Callback<void(const FileCreationInfo&)> done) {
+  if (!enable_disk_) {
+    BlobMemoryController::FileCreationInfo creation_info;
+    creation_info.error = File::FILE_ERROR_FAILED;
+    done.Run(creation_info);
+    return;
+  }
+  disk_used_ += size_bytes;
+  std::string file_name = base::Uint64ToString(current_file_num_++);
+  base::PostTaskAndReplyWithResult(
+      file_runner_.get(), FROM_HERE,
+      base::Bind(&BlobMemoryController::CreateFile, blob_storage_dir_,
+                 file_name, size_bytes),
+      base::Bind(&BlobMemoryController::OnCreateFile, this->AsWeakPtr(),
+                 size_bytes, done));
+}
+
+void BlobMemoryController::NotifyWhenBlobCanBeRequested(
+    uint64_t memory_size,
+    base::Callback<void(bool)> can_request) {
+  if (!enable_disk_) {
+    blob_memory_used_ += memory_size;
+    can_request.Run(true);
+    return;
+  }
+  if (!blobs_waiting_for_paging_.empty()) {
+    blobs_waiting_for_paging_.push_back(
+        std::make_pair(memory_size, can_request));
+    blobs_waiting_for_paging_size_ += memory_size;
+    SchedulePagingUntilWeCanFit(blobs_waiting_for_paging_size_);
+    return;
+  }
+  DCHECK_EQ(0u, blobs_waiting_for_paging_size_);
+  SchedulePagingUntilWeCanFit(memory_size);
+  if (memory_size < GetAvailableMemoryForBlobs()) {
+    blob_memory_used_ += memory_size;
+    can_request.Run(true);
+    return;
+  }
+  blobs_waiting_for_paging_.push_back(std::make_pair(memory_size, can_request));
+  blobs_waiting_for_paging_size_ += memory_size;
+  SchedulePagingUntilWeCanFit(blobs_waiting_for_paging_size_);
+}
+
+void BlobMemoryController::UpdateBlobItemInRecents(
+    ShareableBlobDataItem* item) {
+  if (!enable_disk_)
+    return;
+  auto iterator = recent_item_cache_.Get(item->item_id());
+  if (iterator != recent_item_cache_.end())
+    recent_item_cache_.Erase(iterator);
+  recent_item_cache_.Put(item->item_id(), item);
+}
+
+void BlobMemoryController::RemoveBlobItemInRecents(uint64_t id) {
+  if (!enable_disk_)
+    return;
+  auto iterator = recent_item_cache_.Get(id);
+  if (iterator != recent_item_cache_.end())
+    recent_item_cache_.Erase(iterator);
+}
+
+/* static */
+bool BlobMemoryController::CalculateBlobMemorySize(
+    const std::vector<DataElement>& descriptions,
+    size_t* shortcut_bytes,
+    uint64_t* total_bytes) {
+  DCHECK(shortcut_bytes);
+  DCHECK(total_bytes);
+  base::CheckedNumeric<uint64_t> total_size_checked = 0;
+  base::CheckedNumeric<size_t> shortcut_size_checked = 0;
+  for (const auto& info : descriptions) {
+    if (info.type() == DataElement::TYPE_BYTES) {
+      total_size_checked += info.length();
+      shortcut_size_checked += info.length();
+    } else if (info.type() == DataElement::TYPE_BYTES_DESCRIPTION) {
+      total_size_checked += info.length();
+    } else {
+      continue;
+    }
+
+    if (!total_size_checked.IsValid() || !shortcut_size_checked.IsValid()) {
+      return false;
+    }
+  }
+  *shortcut_bytes = shortcut_size_checked.ValueOrDie();
+  *total_bytes = total_size_checked.ValueOrDie();
+  return true;
+}
+
+/* static */
+BlobMemoryController::FileCreationInfo BlobMemoryController::CreateFile(
+    const FilePath& directory_path,
+    const std::string& file_name,
+    size_t size_bytes) {
+  DCHECK_NE(0u, size_bytes);
+  BlobMemoryController::FileCreationInfo creation_info;
+  UMA_HISTOGRAM_MEMORY_KB("Storage.Blob.RendererFileSize", size_bytes / 1024);
+  creation_info.error = File::FILE_OK;
+  base::CreateDirectoryAndGetError(directory_path, &creation_info.error);
+  UMA_HISTOGRAM_ENUMERATION("Storage.Blob.EnsureBlobStorageDirectory",
+                            -creation_info.error, -File::FILE_ERROR_MAX);
+  if (creation_info.error != File::FILE_OK)
+    return creation_info;
+  base::FilePath path = directory_path.Append(file_name);
+  File file(path, File::FLAG_CREATE_ALWAYS);
+  creation_info.error = file.error_details();
+  UMA_HISTOGRAM_ENUMERATION("Storage.Blob.RendererFileCreate",
+                            -creation_info.error, -File::FILE_ERROR_MAX);
+  if (creation_info.error != File::FILE_OK)
+    return creation_info;
+  File::Info file_info;
+  bool success = file.GetInfo(&file_info);
+  UMA_HISTOGRAM_BOOLEAN("Storage.Blob.RendererFileInfoSuccess", success);
+  creation_info.error = success ? File::FILE_OK : File::FILE_ERROR_FAILED;
+  creation_info.last_modified = file_info.last_modified;
+  if (success) {
+    creation_info.file_reference = ShareableFileReference::GetOrCreate(
+        path, ShareableFileReference::DELETE_ON_FINAL_RELEASE,
+        base::ThreadTaskRunnerHandle::Get().get());
+  }
+  return creation_info;
+}
+
+/* static */
+BlobMemoryController::FileCreationInfo BlobMemoryController::WriteItemsToFile(
+    std::vector<scoped_refptr<ShareableBlobDataItem>>* items,
+    size_t total_size_bytes,
+    const FilePath& directory_path,
+    const std::string& file_name) {
+  DCHECK_NE(0u, total_size_bytes);
+  UMA_HISTOGRAM_MEMORY_KB("Storage.Blob.PageFileSize", total_size_bytes / 1024);
+  BlobMemoryController::FileCreationInfo creation_info;
+  creation_info.error = File::FILE_OK;
+  base::CreateDirectoryAndGetError(directory_path, &creation_info.error);
+  UMA_HISTOGRAM_ENUMERATION("Storage.Blob.EnsureBlobStorageDirectory",
+                            -creation_info.error, -File::FILE_ERROR_MAX);
+  if (creation_info.error != File::FILE_OK)
+    return creation_info;
+
+  base::FilePath path = directory_path.Append(file_name);
+  File file(path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
+  creation_info.error = file.error_details();
+  UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PageFileCreate", -creation_info.error,
+                            -File::FILE_ERROR_MAX);
+  if (creation_info.error != File::FILE_OK)
+    return creation_info;
+
+  file.SetLength(total_size_bytes);
+  int bytes_written = 0;
+  for (const auto& refptr : *items) {
+    const DataElement& element = refptr->item()->data_element();
+    DCHECK_EQ(DataElement::TYPE_BYTES, element.type());
+    int length = base::checked_cast<int, uint64>(element.length());
+    bytes_written = file.WriteAtCurrentPos(element.bytes(), length);
+    DCHECK_EQ(length, bytes_written);
+    if (bytes_written < 0) {
+      break;
+    }
+  }
+  UMA_HISTOGRAM_BOOLEAN("Storage.Blob.PageFileWriteSuccess", bytes_written > 0);
+  File::Info info;
+  bool success = file.GetInfo(&info);
+  UMA_HISTOGRAM_BOOLEAN("Storage.Blob.PageFileInfoSuccess", success);
+  creation_info.error =
+      bytes_written < 0 || !success ? File::FILE_ERROR_FAILED : File::FILE_OK;
+  if (creation_info.error == File::FILE_OK) {
+    creation_info.file_reference = ShareableFileReference::GetOrCreate(
+        path, ShareableFileReference::DELETE_ON_FINAL_RELEASE,
+        base::ThreadTaskRunnerHandle::Get().get());
+  }
+  return creation_info;
+}
+
+void BlobMemoryController::OnCreateFile(
+    uint64_t file_size,
+    base::Callback<void(const FileCreationInfo&)> done,
+    FileCreationInfo result) {
+  if (result.error == File::FILE_OK) {
+    result.file_reference->AddFinalReleaseCallback(base::Bind(
+        &BlobMemoryController::OnBlobFileDelete, this->AsWeakPtr(), file_size));
+  } else {
+    disk_used_ -= file_size;
+  }
+  done.Run(result);
+}
+
+size_t BlobMemoryController::ScheduleBlobPaging() {
+  DCHECK(enable_disk_);
+  DCHECK_LT(min_page_file_size_, static_cast<uint64_t>(blob_memory_used_));
+  size_t total_items_size = 0;
+  scoped_ptr<std::vector<scoped_refptr<ShareableBlobDataItem>>> items_for_disk(
+      new std::vector<scoped_refptr<ShareableBlobDataItem>>());
+  while (total_items_size < min_page_file_size_) {
+    DCHECK(!recent_item_cache_.empty());
+    auto iterator = --recent_item_cache_.end();
+    ShareableBlobDataItem* item = iterator->second;
+    recent_item_cache_.Erase(iterator);
+    size_t size = base::checked_cast<size_t, uint64_t>(item->item()->length());
+    total_items_size += size;
+    items_for_disk->push_back(make_scoped_refptr(item));
+  }
+  disk_used_ += total_items_size;
+  std::string file_name = base::Uint64ToString(current_file_num_++);
+  FilePath path = blob_storage_dir_.Append(file_name);
+  base::PostTaskAndReplyWithResult(
+      file_runner_.get(), FROM_HERE,
+      base::Bind(&BlobMemoryController::WriteItemsToFile, items_for_disk.get(),
+                 total_items_size, blob_storage_dir_, file_name),
+      base::Bind(&BlobMemoryController::OnPagingComplete, this->AsWeakPtr(),
+                 base::Passed(items_for_disk.Pass()), total_items_size));
+
+  return total_items_size;
+}
+
+void BlobMemoryController::SchedulePagingUntilWeCanFit(
+    size_t total_memory_needed) {
+  DCHECK_LT(total_memory_needed, max_blob_memory_space_);
+  while (total_memory_needed + blob_memory_used_ > max_blob_memory_space_) {
+    // schedule another page.
+    size_t memory_freeing = ScheduleBlobPaging();
+    DCHECK_GE(blob_memory_used_, memory_freeing);
+    blob_memory_used_ -= memory_freeing;
+    in_flight_memory_used_ += memory_freeing;
+  }
+}
+
+void BlobMemoryController::OnPagingComplete(
+    scoped_ptr<std::vector<scoped_refptr<ShareableBlobDataItem>>> items,
+    size_t total_items_size,
+    FileCreationInfo result) {
+  if (!enable_disk_) {
+    return;
+  }
+  if (result.error != File::FILE_OK) {
+    UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingError", -result.error,
+                              -File::FILE_ERROR_MAX);
+    disk_used_ -= total_items_size;
+    DisableDisk();
+    return;
+  }
+  // Switch from the data backing item to a new file backing item.
+  uint64_t offset = 0;
+  for (const scoped_refptr<ShareableBlobDataItem>& shareable_item :
+       *items.get()) {
+    scoped_refptr<BlobDataItem> new_item(new BlobDataItem(
+        make_scoped_ptr(new DataElement()), result.file_reference));
+    new_item->data_element_ptr()->SetToFilePathRange(
+        result.file_reference->path(), offset, shareable_item->item()->length(),
+        result.last_modified);
+    shareable_item->item_.swap(new_item);
+    offset += shareable_item->item()->length();
+  }
+  in_flight_memory_used_ -= total_items_size;
+
+  // We want callback on blobs up to the amount we've freed.
+  size_t space_available = GetAvailableMemoryForBlobs();
+  while (!blobs_waiting_for_paging_.empty() &&
+         space_available > blobs_waiting_for_paging_.front().first) {
+    auto size_callback_pair = blobs_waiting_for_paging_.front();
+    blobs_waiting_for_paging_.pop_front();
+    space_available -= size_callback_pair.first;
+    blob_memory_used_ += size_callback_pair.first;
+    size_callback_pair.second.Run(true);
+  }
+}
+
+void BlobMemoryController::DisableDisk() {
+  enable_disk_ = false;
+  blob_memory_used_ += in_flight_memory_used_;
+  in_flight_memory_used_ = 0;
+  for (const auto& size_callback_pair : blobs_waiting_for_paging_) {
+    size_callback_pair.second.Run(false);
+  }
+  blobs_waiting_for_paging_.clear();
+  blobs_waiting_for_paging_size_ = 0;
+  recent_item_cache_.Clear();
+  RecordTracingCounters();
+}
+
+void BlobMemoryController::RecordTracingCounters() {
+  TRACE_COUNTER2("Blob", "MemoryUsage", "RegularStorage", blob_memory_used_,
+                 "InFlightToDisk", in_flight_memory_used_);
+  TRACE_COUNTER1("Blob", "TranfersPendingOnDisk",
+                 blobs_waiting_for_paging_.size());
+  TRACE_COUNTER1("Blob", "TranfersBytesPendingOnDisk",
+                 blobs_waiting_for_paging_size_);
+}
+
+size_t BlobMemoryController::GetAvailableMemoryForBlobs() {
+  if (max_blob_memory_space_ < blob_memory_used_)
+    return 0;
+  if (enable_disk_)
+    return std::min<size_t>(0u, in_flight_space_ - in_flight_memory_used_) +
+           (max_blob_memory_space_ - blob_memory_used_);
+  return max_blob_memory_space_ - blob_memory_used_;
+}
+
+uint64_t BlobMemoryController::GetAvailableDiskSpaceForBlobs() {
+  return enable_disk_ ? kBlobStorageMaxDiskSpace - disk_used_ : 0;
+}
+
+void BlobMemoryController::OnBlobFileDelete(uint64_t size,
+                                            const base::FilePath& path) {
+  if (enable_disk_) {
+    DCHECK_GT(size, disk_used_);
+    disk_used_ -= size;
+  }
+}
+
+}  // namespace storage