[BlobStorage] Enabling disk paging and direct storage.

storage/browser/blob/blob_memory_controller.cc

Index: storage/browser/blob/blob_memory_controller.cc
diff --git a/storage/browser/blob/blob_memory_controller.cc b/storage/browser/blob/blob_memory_controller.cc
index f4a002fce198709621fa48b9224837b755bd62ac..4f585883722b6f4928eabf853f1b58baf63877bf 100644
--- a/storage/browser/blob/blob_memory_controller.cc
+++ b/storage/browser/blob/blob_memory_controller.cc
@@ -20,12 +20,12 @@
 #include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
+#include "base/sys_info.h"
 #include "base/task_runner.h"
 #include "base/task_runner_util.h"
 #include "base/threading/thread_restrictions.h"
 #include "base/time/time.h"
 #include "base/trace_event/trace_event.h"
-#include "base/tuple.h"
 #include "storage/browser/blob/blob_data_builder.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
@@ -37,10 +37,51 @@ using base::FilePath;
 
 namespace storage {
 namespace {
+
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using MemoryAllocation = BlobMemoryController::MemoryAllocation;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
 
+// CrOS:
+// * Ram -  20%

[jsbell, 2016/12/07 00:01:24]

Nit: Capitalize RAM or just say 'Memory'

+// * Disk - 50%
+// Android:
+// * RAM -  20%
+// * Disk -  5%
+// Desktop:
+// * Ram -  20%, or 2 gigs if x64.

[jsbell, 2016/12/07 00:01:24]

Nit: GB or GiB

+// * Disk - 10%

[michaeln, 2016/12/06 21:58:23]

hmmm... i'm not sure how to evaluate these numbers?

[dmurph, 2016/12/06 22:12:43]

Josh, Victor, and I chatted for a while about them. What would you like? I can
clarify that this is relative to device stats.

[michaeln, 2016/12/06 23:11:25]

I'm not sure, i haven't thought much about the policy for this resource
utilization and presented with hard numbers suddenly makes it real.

The amount of freediskspace() comes to mind when considering disk usage. Maybe
whatever 'limit' is chosen should be tempered in light of actual availability
when we're being asked to alloc something.

The 50% cros number concerns me. Storage on my personal chromeos device is
fairly filled up. The blob system probably shouldn't be able to consume 100% of
what's remaining. Would it do that as coded right now?

[dmurph, 2016/12/06 23:20:50]

Free disk space changes, so I didn't want to use that.

We want to accommodate the use case of people using blob-heavy applications -
see
https://docs.google.com/document/d/1vc2YtaqX0j6nkPGmjo7P4ObXqAPT99Q1zxFGbYdnz...
for some examples.

We aren't totally reserving this - this is just the ceiling for blobs.
Chromebooks have not much disk space, so we wanted a high percentage here. If we
hit a full disk, blobs will fail as normal, and they will go away when you close
the tab / blobs are dereferenced.

+BlobStorageLimits CalculateBlobStorageLimitsImpl(const FilePath& storage_dir,
+                                                 bool disk_enabled) {
+  BlobStorageLimits output;
+
+  int64_t disk_size =
+      disk_enabled ? base::SysInfo::AmountOfTotalDiskSpace(storage_dir) : 0ull;
+  int64_t memory_size = base::SysInfo::AmountOfPhysicalMemory();
+
+  BlobStorageLimits limits;
+
+  if (memory_size > 0) {
+#if !defined(OS_CHROMEOS) && !defined(OS_ANDROID) && defined(ARCH_CPU_64_BITS)

[michaeln, 2016/12/06 23:11:25]

is there such a thing as a 64bit system with < 2G of physical memory, would it
make sense to constrain this along the lines of std::min(2G, 20%)?

[dmurph, 2016/12/06 23:20:50]

The reasoning here is that if it's x64, then it has virtual memory and we're not
concerned. We explicitly don't do this on android or chromeos though, which I
don't think has that. So I would even do max(2G, 20%)

+    static const size_t kTwoGigabytes = 2ull * 1024 * 1024 * 1024;
+    limits.max_blob_in_memory_space = kTwoGigabytes;
+#else
+    limits.max_blob_in_memory_space = static_cast<size_t>(memory_size / 5ll);
+#endif
+  }
+
+  if (disk_size > 0) {
+#if defined(OS_CHROMEOS)
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 2ll);
+#elif defined(OS_ANDROID)
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 20ll);
+#else
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 10ll);
+#endif
+  }
+  return limits;
+}
+
 File::Error CreateBlobDirectory(const FilePath& blob_storage_dir) {
   File::Error error = File::FILE_OK;
   base::CreateDirectoryAndGetError(blob_storage_dir, &error);
@@ -421,7 +462,8 @@ BlobMemoryController::Strategy BlobMemoryController::DetermineStrategy(
     return Strategy::NONE_NEEDED;
   }
   if (file_paging_enabled_ &&
-      (total_transportation_bytes > limits_.memory_limit_before_paging())) {
+      total_transportation_bytes <= GetAvailableFileSpaceForBlobs() &&
+      total_transportation_bytes > limits_.memory_limit_before_paging()) {
     return Strategy::FILE;
   }
   if (total_transportation_bytes > limits_.max_ipc_memory_size)
@@ -516,6 +558,29 @@ void BlobMemoryController::NotifyMemoryItemsUsed(
   MaybeScheduleEvictionUntilSystemHealthy();
 }
 
+void BlobMemoryController::CalculateBlobStorageLimits() {
+  if (file_runner_) {
+    PostTaskAndReplyWithResult(
+        file_runner_.get(), FROM_HERE,
+        base::Bind(&CalculateBlobStorageLimitsImpl, blob_storage_dir_, true),
+        base::Bind(&BlobMemoryController::OnStorageLimitsCalculated,
+                   weak_factory_.GetWeakPtr()));
+  } else {
+    OnStorageLimitsCalculated(
+        CalculateBlobStorageLimitsImpl(blob_storage_dir_, false));
+  }
+}
+
+base::WeakPtr<BlobMemoryController> BlobMemoryController::GetWeakPtr() {
+  return weak_factory_.GetWeakPtr();
+}
+
+void BlobMemoryController::OnStorageLimitsCalculated(BlobStorageLimits limits) {
+  if (!limits.IsValid() || manual_limits_set_)
+    return;
+  limits_ = limits;
+}
+
 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::AppendMemoryTask(
     uint64_t total_bytes_needed,
     std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,