Add memory pressure listener to Blob storage

storage/browser/blob/blob_memory_controller.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_memory_controller.h"
 
 #include <algorithm>
 #include <numeric>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/callback_helpers.h"
 #include "base/containers/small_map.h"
 #include "base/files/file_util.h"
 #include "base/guid.h"
 #include "base/location.h"
 #include "base/memory/ptr_util.h"
+#include "base/metrics/histogram_functions.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/numerics/safe_math.h"
 #include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/sys_info.h"
 #include "base/task_runner.h"
 #include "base/task_runner_util.h"
 #include "base/threading/thread_restrictions.h"
 #include "base/time/time.h"
 #include "base/trace_event/trace_event.h"
 #include "storage/browser/blob/blob_data_builder.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
 #include "storage/browser/blob/shareable_file_reference.h"
 #include "storage/common/data_element.h"
 
 using base::File;
 using base::FilePath;
 
 namespace storage {
 namespace {
 constexpr int64_t kUnknownDiskAvailability = -1ll;
 constexpr uint64_t kMegabyte = 1024ull * 1024;
+const int64_t kMinSecondsForPressureEvictions = 30;
 
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using MemoryAllocation = BlobMemoryController::MemoryAllocation;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
 using DiskSpaceFuncPtr = BlobMemoryController::DiskSpaceFuncPtr;
 
 // CrOS:
 // * Ram -  20%
 // * Disk - 50%
 //   Note: The disk is the user partition, so the operating system can still
@@ skipping 444 matching lines @@
 
 BlobMemoryController::BlobMemoryController(
     const base::FilePath& storage_directory,
     scoped_refptr<base::TaskRunner> file_runner)
     : file_paging_enabled_(file_runner.get() != nullptr),
       blob_storage_dir_(storage_directory),
       file_runner_(std::move(file_runner)),
       disk_space_function_(&base::SysInfo::AmountOfFreeDiskSpace),
       populated_memory_items_(
           base::MRUCache<uint64_t, ShareableBlobDataItem*>::NO_AUTO_EVICT),
+      memory_pressure_listener_(
+          base::Bind(&BlobMemoryController::OnMemoryPressure,
+                     base::Unretained(this))),
       weak_factory_(this) {}
 
 BlobMemoryController::~BlobMemoryController() {}
 
 void BlobMemoryController::DisableFilePaging(base::File::Error reason) {
   UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingDisabled", -reason,
                             -File::FILE_ERROR_MAX);
   DLOG(ERROR) << "Blob storage paging disabled, reason: " << reason;
   file_paging_enabled_ = false;
   in_flight_memory_used_ = 0;
@@ skipping 79 matching lines @@
   // more paging for any more pending blobs.
   if (!pending_memory_quota_tasks_.empty()) {
     return AppendMemoryTask(total_bytes_needed,
                             std::move(unreserved_memory_items), done_callback);
   }
 
   // Store right away if we can.
   if (total_bytes_needed <= GetAvailableMemoryForBlobs()) {
     GrantMemoryAllocations(&unreserved_memory_items,
                            static_cast<size_t>(total_bytes_needed));
-    MaybeScheduleEvictionUntilSystemHealthy();
+    MaybeScheduleEvictionUntilSystemHealthy(
+        base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
     done_callback.Run(true);
     return base::WeakPtr<QuotaAllocationTask>();
   }
 
   // Size is larger than available memory.
   DCHECK(pending_memory_quota_tasks_.empty());
   DCHECK_EQ(0u, pending_memory_quota_total_size_);
 
   auto weak_ptr = AppendMemoryTask(
       total_bytes_needed, std::move(unreserved_memory_items), done_callback);
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
   return weak_ptr;
 }
 
 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveFileQuota(
     std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
     const FileQuotaRequestCallback& done_callback) {
   pending_file_quota_tasks_.push_back(base::MakeUnique<FileQuotaAllocationTask>(
       this, disk_space_function_, std::move(unreserved_file_items),
       done_callback));
   pending_file_quota_tasks_.back()->set_my_list_position(
@@ skipping 13 matching lines @@
         items_paging_to_file_.end()) {
       return;
     }
     auto iterator = populated_memory_items_.Get(item->item_id());
     if (iterator == populated_memory_items_.end()) {
       populated_memory_items_bytes_ +=
           static_cast<size_t>(item->item()->length());
       populated_memory_items_.Put(item->item_id(), item.get());
     }
   }
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
 }
 
 void BlobMemoryController::CalculateBlobStorageLimits() {
   if (file_runner_) {
     PostTaskAndReplyWithResult(
         file_runner_.get(), FROM_HERE,
         base::Bind(&CalculateBlobStorageLimitsImpl, blob_storage_dir_, true),
         base::Bind(&BlobMemoryController::OnStorageLimitsCalculated,
                    weak_factory_.GetWeakPtr()));
   } else {
@@ skipping 101 matching lines @@
     std::unique_ptr<MemoryQuotaAllocationTask> memory_task =
         std::move(pending_memory_quota_tasks_.front());
     pending_memory_quota_tasks_.pop_front();
     pending_memory_quota_total_size_ -= memory_task->allocation_size();
     memory_task->RunDoneCallback(true);
   }
   RecordTracingCounters();
 }
 
 size_t BlobMemoryController::CollectItemsForEviction(
-    std::vector<scoped_refptr<ShareableBlobDataItem>>* output) {
+    std::vector<scoped_refptr<ShareableBlobDataItem>>* output,
+    uint64_t min_page_file_size) {
   base::CheckedNumeric<size_t> total_items_size = 0;
   // Process the recent item list and remove items until we have at least a
   // minimum file size or we're at the end of our items to page to disk.
-  while (total_items_size.ValueOrDie() < limits_.min_page_file_size &&
+  while (total_items_size.ValueOrDie() < min_page_file_size &&
          !populated_memory_items_.empty()) {
     auto iterator = --populated_memory_items_.end();
     ShareableBlobDataItem* item = iterator->second;
     DCHECK_EQ(item->item()->type(), DataElement::TYPE_BYTES);
     populated_memory_items_.Erase(iterator);
     size_t size = base::checked_cast<size_t>(item->item()->length());
     populated_memory_items_bytes_ -= size;
     total_items_size += size;
     output->push_back(make_scoped_refptr(item));
   }
   return total_items_size.ValueOrDie();
 }
 
-void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
+void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
   // Don't do eviction when others are happening, as we don't change our
   // pending_memory_quota_total_size_ value until after the paging files have
   // been written.
   if (pending_evictions_ != 0 || !file_paging_enabled_)
     return;
 
   uint64_t total_memory_usage =
       static_cast<uint64_t>(pending_memory_quota_total_size_) +
       blob_memory_used_;
 
+  size_t in_memory_limit = limits_.memory_limit_before_paging();
+  uint64_t min_page_file_size = limits_.min_page_file_size;
+  if (memory_pressure_level !=
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
+    in_memory_limit = 0;
+    // Use lower page file size to reduce using more memory for writing under
+    // pressure.
+    min_page_file_size = limits_.max_blob_in_memory_space *
+                         limits_.max_blob_in_memory_space_under_pressure_ratio;
+  }
+
   // We try to page items to disk until our current system size + requested
   // memory is below our size limit.
   // Size limit is a lower |memory_limit_before_paging()| if we have disk space.
   while (total_memory_usage > limits_.effective_max_disk_space ||
          (disk_used_ < limits_.effective_max_disk_space &&
-          total_memory_usage > limits_.memory_limit_before_paging())) {
+          total_memory_usage > in_memory_limit)) {
+    const char* reason = nullptr;
+    if (memory_pressure_level !=
+        base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
+      reason = "OnMemoryPressure";
+    } else if (total_memory_usage > limits_.effective_max_disk_space) {
+      reason = "SizeExceededMaxDiskSpace";
+    } else {
+      reason = "SizeExceededInMemoryLimit";
+    }
+
     // We only page when we have enough items to fill a whole page file.
-    if (populated_memory_items_bytes_ < limits_.min_page_file_size)
+    if (populated_memory_items_bytes_ < min_page_file_size)
       break;
-    DCHECK_LE(limits_.min_page_file_size,
-              static_cast<uint64_t>(blob_memory_used_));
+    DCHECK_LE(min_page_file_size, static_cast<uint64_t>(blob_memory_used_));
 
     std::vector<scoped_refptr<ShareableBlobDataItem>> items_to_swap;
-    size_t total_items_size = CollectItemsForEviction(&items_to_swap);
+
+    size_t total_items_size =
+        CollectItemsForEviction(&items_to_swap, min_page_file_size);
     if (total_items_size == 0)
       break;
 
     std::vector<DataElement*> items_for_paging;
     for (auto& shared_blob_item : items_to_swap) {
       items_paging_to_file_.insert(shared_blob_item->item_id());
       items_for_paging.push_back(shared_blob_item->item()->data_element_ptr());
     }
 
     // Update our bookkeeping.
@@ skipping 15 matching lines @@
 
     // Post the file writing task.
     base::PostTaskAndReplyWithResult(
         file_runner_.get(), FROM_HERE,
         base::Bind(&CreateFileAndWriteItems, blob_storage_dir_,
                    disk_space_function_, base::Passed(&page_file_path),
                    file_runner_, base::Passed(&items_for_paging),
                    total_items_size),
         base::Bind(&BlobMemoryController::OnEvictionComplete,
                    weak_factory_.GetWeakPtr(), base::Passed(&file_reference),
-                   base::Passed(&items_to_swap), total_items_size));
+                   base::Passed(&items_to_swap), total_items_size, reason,
+                   total_memory_usage));
+
+    last_eviction_time_ = base::TimeTicks::Now();
   }
   RecordTracingCounters();
 }
 
 void BlobMemoryController::OnEvictionComplete(
     scoped_refptr<ShareableFileReference> file_reference,
     std::vector<scoped_refptr<ShareableBlobDataItem>> items,
     size_t total_items_size,
+    const char* evict_reason,
+    size_t memory_usage_before_eviction,
     std::pair<FileCreationInfo, int64_t /* avail_disk */> result) {
   if (!file_paging_enabled_)
     return;
 
   FileCreationInfo& file_info = std::get<0>(result);
   int64_t avail_disk_space = std::get<1>(result);
 
   if (file_info.error != File::FILE_OK) {
     DisableFilePaging(file_info.error);
     return;
@@ skipping 15 matching lines @@
         file_reference->path(), offset, shareable_item->item()->length(),
         file_info.last_modified);
     DCHECK(shareable_item->memory_allocation_);
     shareable_item->set_memory_allocation(nullptr);
     shareable_item->set_item(new_item);
     items_paging_to_file_.erase(shareable_item->item_id());
     offset += shareable_item->item()->length();
   }
   in_flight_memory_used_ -= total_items_size;
 
+  // Record change in memory usage at the last eviction reply.
+  size_t total_usage = blob_memory_used_ + pending_memory_quota_total_size_;
+  if (!pending_evictions_ && memory_usage_before_eviction >= total_usage) {
+    std::string full_histogram_name =
+        std::string("Storage.Blob.SizeEvictedToDiskInKB.") + evict_reason;
+    base::UmaHistogramCounts100000(
+        full_histogram_name,
+        (memory_usage_before_eviction - total_usage) / 1024);
+  }
+
   // We want callback on blobs up to the amount we've freed.
   MaybeGrantPendingMemoryRequests();
 
   // If we still have more blobs waiting and we're not waiting on more paging
   // operations, schedule more.
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
+}
+
+void BlobMemoryController::OnMemoryPressure(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
+  auto time_from_last_evicion = base::TimeTicks::Now() - last_eviction_time_;
+  if (time_from_last_evicion.InSeconds() < kMinSecondsForPressureEvictions)
+    return;
+
+  MaybeScheduleEvictionUntilSystemHealthy(memory_pressure_level);
 }
 
 FilePath BlobMemoryController::GenerateNextPageFileName() {
   std::string file_name = base::Uint64ToString(current_file_num_++);
   return blob_storage_dir_.Append(FilePath::FromUTF8Unsafe(file_name));
 }
 
 void BlobMemoryController::RecordTracingCounters() const {
   TRACE_COUNTER2("Blob", "MemoryUsage", "TotalStorage", blob_memory_used_,
                  "InFlightToDisk", in_flight_memory_used_);
@@ skipping 65 matching lines @@
   MaybeGrantPendingMemoryRequests();
 }
 
 void BlobMemoryController::OnBlobFileDelete(uint64_t size,
                                             const FilePath& path) {
   DCHECK_LE(size, disk_used_);
   disk_used_ -= size;
 }
 
 }  // namespace storage