[BlobStorage] Enabling disk paging and direct storage.

storage/browser/blob/blob_memory_controller.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_memory_controller.h"
 
 #include <algorithm>
 #include <numeric>
 
 #include "base/callback.h"
 #include "base/callback_helpers.h"
 #include "base/containers/small_map.h"
 #include "base/files/file_util.h"
 #include "base/guid.h"
 #include "base/location.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/numerics/safe_math.h"
 #include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
+#include "base/sys_info.h"
 #include "base/task_runner.h"
 #include "base/task_runner_util.h"
 #include "base/threading/thread_restrictions.h"
 #include "base/time/time.h"
 #include "base/trace_event/trace_event.h"
-#include "base/tuple.h"
 #include "storage/browser/blob/blob_data_builder.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
 #include "storage/browser/blob/shareable_file_reference.h"
 #include "storage/common/data_element.h"
 
 using base::File;
 using base::FilePath;
 
 namespace storage {
 namespace {
+
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using MemoryAllocation = BlobMemoryController::MemoryAllocation;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
 
+// CrOS:
+// * Ram -  20%

[jsbell, 2016/12/07 00:01:24]

Nit: Capitalize RAM or just say 'Memory'

+// * Disk - 50%
+// Android:
+// * RAM -  20%
+// * Disk -  5%
+// Desktop:
+// * Ram -  20%, or 2 gigs if x64.

[jsbell, 2016/12/07 00:01:24]

Nit: GB or GiB

+// * Disk - 10%

[michaeln, 2016/12/06 21:58:23]

hmmm... i'm not sure how to evaluate these numbers?

[dmurph, 2016/12/06 22:12:43]

Josh, Victor, and I chatted for a while about them. What would you like? I can
clarify that this is relative to device stats.

[michaeln, 2016/12/06 23:11:25]

I'm not sure, i haven't thought much about the policy for this resource
utilization and presented with hard numbers suddenly makes it real.

The amount of freediskspace() comes to mind when considering disk usage. Maybe
whatever 'limit' is chosen should be tempered in light of actual availability
when we're being asked to alloc something.

The 50% cros number concerns me. Storage on my personal chromeos device is
fairly filled up. The blob system probably shouldn't be able to consume 100% of
what's remaining. Would it do that as coded right now?

[dmurph, 2016/12/06 23:20:50]

Free disk space changes, so I didn't want to use that.

We want to accommodate the use case of people using blob-heavy applications -
see
https://docs.google.com/document/d/1vc2YtaqX0j6nkPGmjo7P4ObXqAPT99Q1zxFGbYdnz...
for some examples.

We aren't totally reserving this - this is just the ceiling for blobs.
Chromebooks have not much disk space, so we wanted a high percentage here. If we
hit a full disk, blobs will fail as normal, and they will go away when you close
the tab / blobs are dereferenced.

+BlobStorageLimits CalculateBlobStorageLimitsImpl(const FilePath& storage_dir,
+                                                 bool disk_enabled) {
+  BlobStorageLimits output;
+
+  int64_t disk_size =
+      disk_enabled ? base::SysInfo::AmountOfTotalDiskSpace(storage_dir) : 0ull;
+  int64_t memory_size = base::SysInfo::AmountOfPhysicalMemory();
+
+  BlobStorageLimits limits;
+
+  if (memory_size > 0) {
+#if !defined(OS_CHROMEOS) && !defined(OS_ANDROID) && defined(ARCH_CPU_64_BITS)

[michaeln, 2016/12/06 23:11:25]

is there such a thing as a 64bit system with < 2G of physical memory, would it
make sense to constrain this along the lines of std::min(2G, 20%)?

[dmurph, 2016/12/06 23:20:50]

The reasoning here is that if it's x64, then it has virtual memory and we're not
concerned. We explicitly don't do this on android or chromeos though, which I
don't think has that. So I would even do max(2G, 20%)

+    static const size_t kTwoGigabytes = 2ull * 1024 * 1024 * 1024;
+    limits.max_blob_in_memory_space = kTwoGigabytes;
+#else
+    limits.max_blob_in_memory_space = static_cast<size_t>(memory_size / 5ll);
+#endif
+  }
+
+  if (disk_size > 0) {
+#if defined(OS_CHROMEOS)
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 2ll);
+#elif defined(OS_ANDROID)
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 20ll);
+#else
+    limits.max_blob_disk_space = static_cast<size_t>(disk_size / 10ll);
+#endif
+  }
+  return limits;
+}
+
 File::Error CreateBlobDirectory(const FilePath& blob_storage_dir) {
   File::Error error = File::FILE_OK;
   base::CreateDirectoryAndGetError(blob_storage_dir, &error);
   UMA_HISTOGRAM_ENUMERATION("Storage.Blob.CreateDirectoryResult", -error,
                             -File::FILE_ERROR_MAX);
   DLOG_IF(ERROR, error != File::FILE_OK)
       << "Error creating blob storage directory: " << error;
   return error;
 }
 
@@ skipping 360 matching lines @@
     return Strategy::TOO_LARGE;
 
   // Handle the case where we have all the bytes preemptively transported, and
   // we can also fit them.
   if (preemptive_transported_bytes == total_transportation_bytes &&
       pending_memory_quota_tasks_.empty() &&
       preemptive_transported_bytes < GetAvailableMemoryForBlobs()) {
     return Strategy::NONE_NEEDED;
   }
   if (file_paging_enabled_ &&
-      (total_transportation_bytes > limits_.memory_limit_before_paging())) {
+      total_transportation_bytes <= GetAvailableFileSpaceForBlobs() &&
+      total_transportation_bytes > limits_.memory_limit_before_paging()) {
     return Strategy::FILE;
   }
   if (total_transportation_bytes > limits_.max_ipc_memory_size)
     return Strategy::SHARED_MEMORY;
   return Strategy::IPC;
 }
 
 bool BlobMemoryController::CanReserveQuota(uint64_t size) const {
   // We check each size independently as a blob can't be constructed in both
   // disk and memory.
@@ skipping 74 matching lines @@
     auto iterator = populated_memory_items_.Get(item->item_id());
     if (iterator == populated_memory_items_.end()) {
       populated_memory_items_bytes_ +=
           static_cast<size_t>(item->item()->length());
       populated_memory_items_.Put(item->item_id(), item.get());
     }
   }
   MaybeScheduleEvictionUntilSystemHealthy();
 }
 
+void BlobMemoryController::CalculateBlobStorageLimits() {
+  if (file_runner_) {
+    PostTaskAndReplyWithResult(
+        file_runner_.get(), FROM_HERE,
+        base::Bind(&CalculateBlobStorageLimitsImpl, blob_storage_dir_, true),
+        base::Bind(&BlobMemoryController::OnStorageLimitsCalculated,
+                   weak_factory_.GetWeakPtr()));
+  } else {
+    OnStorageLimitsCalculated(
+        CalculateBlobStorageLimitsImpl(blob_storage_dir_, false));
+  }
+}
+
+base::WeakPtr<BlobMemoryController> BlobMemoryController::GetWeakPtr() {
+  return weak_factory_.GetWeakPtr();
+}
+
+void BlobMemoryController::OnStorageLimitsCalculated(BlobStorageLimits limits) {
+  if (!limits.IsValid() || manual_limits_set_)
+    return;
+  limits_ = limits;
+}
+
 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::AppendMemoryTask(
     uint64_t total_bytes_needed,
     std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
     const MemoryQuotaRequestCallback& done_callback) {
   DCHECK(file_paging_enabled_)
       << "Caller tried to reserve memory when CanReserveQuota("
       << total_bytes_needed << ") would have returned false.";
 
   pending_memory_quota_total_size_ += total_bytes_needed;
   pending_memory_quota_tasks_.push_back(
@@ skipping 212 matching lines @@
   MaybeGrantPendingMemoryRequests();
 }
 
 void BlobMemoryController::OnBlobFileDelete(uint64_t size,
                                             const FilePath& path) {
   DCHECK_LE(size, disk_used_);
   disk_used_ -= size;
 }
 
 }  // namespace storage