[BlobStorage] Enabling disk paging and direct storage.

storage/browser/blob/blob_memory_controller.cc

Index: storage/browser/blob/blob_memory_controller.cc
diff --git a/storage/browser/blob/blob_memory_controller.cc b/storage/browser/blob/blob_memory_controller.cc
index f4a002fce198709621fa48b9224837b755bd62ac..54253547e0c3038b4484145b5d9823eb7fff18f9 100644
--- a/storage/browser/blob/blob_memory_controller.cc
+++ b/storage/browser/blob/blob_memory_controller.cc
@@ -7,6 +7,8 @@
 #include <algorithm>
 #include <numeric>
 
+#include "base/bind.h"
+#include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/callback_helpers.h"
 #include "base/containers/small_map.h"
@@ -20,12 +22,12 @@
 #include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
+#include "base/sys_info.h"
 #include "base/task_runner.h"
 #include "base/task_runner_util.h"
 #include "base/threading/thread_restrictions.h"
 #include "base/time/time.h"
 #include "base/trace_event/trace_event.h"
-#include "base/tuple.h"
 #include "storage/browser/blob/blob_data_builder.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
@@ -37,10 +39,53 @@ using base::FilePath;
 
 namespace storage {
 namespace {
+const int64_t kUnknownDiskAvailability = -1ll;
+
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using MemoryAllocation = BlobMemoryController::MemoryAllocation;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
 
+// CrOS:
+// * Ram -  20%
+// * Disk - 50%
+// Android:
+// * RAM -  20%
+// * Disk -  5%
+// Desktop:
+// * Ram -  20%, or 2 gigs if x64.
+// * Disk - 10%
+BlobStorageLimits CalculateBlobStorageLimitsImpl(const FilePath& storage_dir,
+                                                 bool disk_enabled) {
+  BlobStorageLimits output;
+
+  int64_t disk_size =
+      disk_enabled ? base::SysInfo::AmountOfTotalDiskSpace(storage_dir) : 0ull;
+  int64_t memory_size = base::SysInfo::AmountOfPhysicalMemory();
+
+  BlobStorageLimits limits;
+
+  if (memory_size > 0) {
+#if !defined(OS_CHROMEOS) && !defined(OS_ANDROID) && defined(ARCH_CPU_64_BITS)
+    static const size_t kTwoGigabytes = 2ull * 1024 * 1024 * 1024;
+    limits.max_blob_in_memory_space = kTwoGigabytes;
+#else
+    limits.max_blob_in_memory_space = static_cast<size_t>(memory_size / 5ll);
+#endif
+  }
+
+  if (disk_size > 0) {
+#if defined(OS_CHROMEOS)
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 2ll);
+#elif defined(OS_ANDROID)
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 20ll);
+#else
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 10ll);
+#endif
+  }
+
+  return limits;
+}
+
 File::Error CreateBlobDirectory(const FilePath& blob_storage_dir) {
   File::Error error = File::FILE_OK;
   base::CreateDirectoryAndGetError(blob_storage_dir, &error);
@@ -53,17 +98,29 @@ File::Error CreateBlobDirectory(const FilePath& blob_storage_dir) {
 
 void DestructFile(File infos_without_references) {}
 
+void DeleteFiles(std::vector<FileCreationInfo> files) {
+  for (FileCreationInfo& file_info : files) {
+    file_info.file.Close();
+    base::DeleteFile(file_info.path, false);
+  }
+}
+
 // Used for new unpopulated file items. Caller must populate file reference in
 // returned FileCreationInfos.
-std::pair<std::vector<FileCreationInfo>, File::Error> CreateEmptyFiles(
-    const FilePath& blob_storage_dir,
-    scoped_refptr<base::TaskRunner> file_task_runner,
-    std::vector<base::FilePath> file_paths) {
+std::tuple<std::vector<FileCreationInfo>, File::Error, int64_t>
+CreateEmptyFiles(const FilePath& blob_storage_dir,
+                 scoped_refptr<base::TaskRunner> file_task_runner,
+                 std::vector<base::FilePath> file_paths) {
   base::ThreadRestrictions::AssertIOAllowed();
 
   File::Error dir_create_status = CreateBlobDirectory(blob_storage_dir);
-  if (dir_create_status != File::FILE_OK)
-    return std::make_pair(std::vector<FileCreationInfo>(), dir_create_status);
+  if (dir_create_status != File::FILE_OK) {
+    return std::make_tuple(std::vector<FileCreationInfo>(), dir_create_status,
+                           kUnknownDiskAvailability);
+  }
+
+  int64_t free_disk_space =
+      base::SysInfo::AmountOfFreeDiskSpace(blob_storage_dir);
 
   std::vector<FileCreationInfo> result;
   for (const base::FilePath& file_path : file_paths) {
@@ -74,19 +131,20 @@ std::pair<std::vector<FileCreationInfo>, File::Error> CreateEmptyFiles(
     creation_info.file_deletion_runner = file_task_runner;
     creation_info.error = file.error_details();
     if (creation_info.error != File::FILE_OK) {
-      return std::make_pair(std::vector<FileCreationInfo>(),
-                            creation_info.error);
+      return std::make_tuple(std::vector<FileCreationInfo>(),
+                             creation_info.error, free_disk_space);
     }
     creation_info.file = std::move(file);
 
     result.push_back(std::move(creation_info));
   }
-  return std::make_pair(std::move(result), File::FILE_OK);
+  return std::make_tuple(std::move(result), File::FILE_OK, free_disk_space);
 }
 
 // Used to evict multiple memory items out to a single file. Caller must
-// populate file reference in returned FileCreationInfo.
-FileCreationInfo CreateFileAndWriteItems(
+// populate file reference in returned FileCreationInfo. Also returns the disk
+// space AFTER creating this file.
+std::pair<FileCreationInfo, int64_t> CreateFileAndWriteItems(
     const FilePath& blob_storage_dir,
     const FilePath& file_path,
     scoped_refptr<base::TaskRunner> file_task_runner,
@@ -100,14 +158,21 @@ FileCreationInfo CreateFileAndWriteItems(
   creation_info.file_deletion_runner = std::move(file_task_runner);
   creation_info.error = CreateBlobDirectory(blob_storage_dir);
   if (creation_info.error != File::FILE_OK)
-    return creation_info;
+    return std::make_pair(std::move(creation_info), kUnknownDiskAvailability);
+
+  int64_t free_disk_space =
+      base::SysInfo::AmountOfFreeDiskSpace(blob_storage_dir);
+  int64_t disk_availability =
+      free_disk_space == kUnknownDiskAvailability
+          ? kUnknownDiskAvailability
+          : free_disk_space - static_cast<int64_t>(total_size_bytes);
 
   // Create the page file.
   File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
   creation_info.path = file_path;
   creation_info.error = file.error_details();
   if (creation_info.error != File::FILE_OK)
-    return creation_info;
+    return std::make_pair(std::move(creation_info), disk_availability);
 
   // Write data.
   file.SetLength(total_size_bytes);
@@ -130,7 +195,7 @@ FileCreationInfo CreateFileAndWriteItems(
   }
   if (!file.Flush()) {
     creation_info.error = File::FILE_ERROR_FAILED;
-    return creation_info;
+    return std::make_pair(std::move(creation_info), disk_availability);
   }
 
   File::Info info;
@@ -138,7 +203,7 @@ FileCreationInfo CreateFileAndWriteItems(
   creation_info.error =
       bytes_written < 0 || !success ? File::FILE_ERROR_FAILED : File::FILE_OK;
   creation_info.last_modified = info.last_modified;
-  return creation_info;
+  return std::make_pair(std::move(creation_info), disk_availability);
 }
 
 uint64_t GetTotalSizeAndFileSizes(
@@ -295,7 +360,8 @@ class BlobMemoryController::FileQuotaAllocationTask
         base::Bind(&CreateEmptyFiles, controller_->blob_storage_dir_,
                    controller_->file_runner_, base::Passed(&file_paths)),
         base::Bind(&FileQuotaAllocationTask::OnCreateEmptyFiles,
-                   weak_factory_.GetWeakPtr(), base::Passed(&references)));
+                   weak_factory_.GetWeakPtr(), base::Passed(&references),
+                   allocation_size_));
     controller_->RecordTracingCounters();
   }
   ~FileQuotaAllocationTask() override {}
@@ -330,15 +396,40 @@ class BlobMemoryController::FileQuotaAllocationTask
 
   void OnCreateEmptyFiles(
       std::vector<scoped_refptr<ShareableFileReference>> references,
-      std::pair<std::vector<FileCreationInfo>, File::Error> files_and_error) {
-    auto& files = files_and_error.first;
+      uint64_t new_files_total_size,
+      std::tuple<std::vector<FileCreationInfo>, File::Error, int64_t> result) {
+    std::vector<FileCreationInfo> files = std::move(std::get<0>(result));
+    File::Error error = std::get<1>(result);
+    int64_t avail_disk_space = std::get<2>(result);
     if (files.empty()) {
+      DCHECK_NE(error, File::FILE_OK);
       DCHECK_GE(controller_->disk_used_, allocation_size_);
       controller_->disk_used_ -= allocation_size_;
       // This will call our callback and delete the object correctly.
-      controller_->DisableFilePaging(files_and_error.second);
+      controller_->DisableFilePaging(error);
+      return;
+    }
+    // The allocation won't fit at all. Freeze the disk and cancel this request.
+    if (avail_disk_space != kUnknownDiskAvailability &&
+        base::checked_cast<uint64_t>(avail_disk_space) < new_files_total_size) {
+      DCHECK_GE(controller_->disk_used_, allocation_size_);
+      controller_->disk_used_ -= allocation_size_;
+      controller_->FreezeDiskUsage(static_cast<uint64_t>(avail_disk_space));
+      file_runner_->PostTask(FROM_HERE,
+                             base::Bind(&DeleteFiles, base::Passed(&files)));
+      std::unique_ptr<FileQuotaAllocationTask> this_object =
+          std::move(*my_list_position_);
+      controller_->pending_file_quota_tasks_.erase(my_list_position_);
+      RunDoneCallback(std::vector<FileCreationInfo>(), false);
       return;
     }
+    // Freeze disk if we're getting close.
+    if (avail_disk_space != kUnknownDiskAvailability &&
+        avail_disk_space - new_files_total_size <=
+            controller_->limits().min_available_disk_space()) {
+      controller_->FreezeDiskUsage(base::checked_cast<uint64_t>(
+          avail_disk_space - new_files_total_size));
+    }
     DCHECK_EQ(files.size(), references.size());
     for (size_t i = 0; i < files.size(); i++) {
       files[i].file_reference = std::move(references[i]);
@@ -417,11 +508,12 @@ BlobMemoryController::Strategy BlobMemoryController::DetermineStrategy(
   // we can also fit them.
   if (preemptive_transported_bytes == total_transportation_bytes &&
       pending_memory_quota_tasks_.empty() &&
-      preemptive_transported_bytes < GetAvailableMemoryForBlobs()) {
+      preemptive_transported_bytes <= GetAvailableMemoryForBlobs()) {
     return Strategy::NONE_NEEDED;
   }
   if (file_paging_enabled_ &&
-      (total_transportation_bytes > limits_.memory_limit_before_paging())) {
+      total_transportation_bytes <= GetAvailableFileSpaceForBlobs() &&
+      total_transportation_bytes > limits_.memory_limit_before_paging()) {
     return Strategy::FILE;
   }
   if (total_transportation_bytes > limits_.max_ipc_memory_size)
@@ -516,6 +608,37 @@ void BlobMemoryController::NotifyMemoryItemsUsed(
   MaybeScheduleEvictionUntilSystemHealthy();
 }
 
+void BlobMemoryController::CalculateBlobStorageLimits() {
+  if (file_runner_) {
+    PostTaskAndReplyWithResult(
+        file_runner_.get(), FROM_HERE,
+        base::Bind(&CalculateBlobStorageLimitsImpl, blob_storage_dir_, true),
+        base::Bind(&BlobMemoryController::OnStorageLimitsCalculated,
+                   weak_factory_.GetWeakPtr()));
+  } else {

[michaeln, 2016/12/08 21:17:33]

it'd be nice if this branch wasn't needed, i guess its for testing is that
right? could the tests provide a 'file_runner_'?

[dmurph, 2016/12/20 02:21:36]

Chatted about. This is to differentiate between incognito (Disk disabled) and
regular modes.

+    OnStorageLimitsCalculated(
+        CalculateBlobStorageLimitsImpl(blob_storage_dir_, false));
+  }
+}
+
+base::WeakPtr<BlobMemoryController> BlobMemoryController::GetWeakPtr() {
+  return weak_factory_.GetWeakPtr();
+}
+
+void BlobMemoryController::OnStorageLimitsCalculated(BlobStorageLimits limits) {
+  if (!limits.IsValid() || manual_limits_set_)
+    return;
+  limits_ = limits;
+}
+
+void BlobMemoryController::FreezeDiskUsage(uint64_t avail_disk) {
+  limits_.max_blob_disk_space = disk_used_;
+  if (avail_disk > limits_.min_available_disk_space()) {
+    limits_.max_blob_disk_space +=
+        avail_disk - limits_.min_available_disk_space();
+  }
+}
+
 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::AppendMemoryTask(
     uint64_t total_bytes_needed,
     std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
@@ -574,10 +697,16 @@ void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
   if (pending_evictions_ != 0 || !file_paging_enabled_)
     return;
 
+  uint64_t total_memory_usage =
+      static_cast<uint64_t>(pending_memory_quota_total_size_) +
+      blob_memory_used_;
+
   // We try to page items to disk until our current system size + requested
   // memory is below our size limit.
-  while (pending_memory_quota_total_size_ + blob_memory_used_ >
-         limits_.memory_limit_before_paging()) {
+  // Size limit is a lower |memory_limit_before_paging()| if we have disk space.
+  while (total_memory_usage > limits_.max_blob_disk_space ||
+         (disk_used_ < limits_.max_blob_disk_space &&
+          total_memory_usage > limits_.memory_limit_before_paging())) {
     // We only page when we have enough items to fill a whole page file.
     if (populated_memory_items_bytes_ < limits_.min_page_file_size)
       break;
@@ -629,15 +758,24 @@ void BlobMemoryController::OnEvictionComplete(
     scoped_refptr<ShareableFileReference> file_reference,
     std::vector<scoped_refptr<ShareableBlobDataItem>> items,
     size_t total_items_size,
-    FileCreationInfo result) {
+    std::pair<FileCreationInfo, int64_t> result) {
   if (!file_paging_enabled_)
     return;
 
-  if (result.error != File::FILE_OK) {
-    DisableFilePaging(result.error);
+  FileCreationInfo& file_info = std::get<0>(result);
+  int64_t avail_disk_space = std::get<1>(result);
+
+  if (file_info.error != File::FILE_OK) {
+    DisableFilePaging(file_info.error);
     return;
   }
 
+  if (avail_disk_space != kUnknownDiskAvailability &&
+      base::checked_cast<uint64_t>(avail_disk_space) <=
+          limits_.min_available_disk_space()) {
+    FreezeDiskUsage(static_cast<uint64_t>(avail_disk_space));
+  }
+
   DCHECK_LT(0, pending_evictions_);
   pending_evictions_--;
 
@@ -648,7 +786,7 @@ void BlobMemoryController::OnEvictionComplete(
         new BlobDataItem(base::WrapUnique(new DataElement()), file_reference));
     new_item->data_element_ptr()->SetToFilePathRange(
         file_reference->path(), offset, shareable_item->item()->length(),
-        result.last_modified);
+        file_info.last_modified);
     DCHECK(shareable_item->memory_allocation_);
     shareable_item->set_memory_allocation(nullptr);
     shareable_item->set_item(new_item);