Add memory pressure listener to Blob storage

storage/browser/blob/blob_memory_controller.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_memory_controller.h"
 
 #include <algorithm>
 #include <numeric>
 
 #include "base/bind.h"
@@ skipping 23 matching lines @@
 #include "storage/browser/blob/shareable_file_reference.h"
 #include "storage/common/data_element.h"
 
 using base::File;
 using base::FilePath;
 
 namespace storage {
 namespace {
 constexpr int64_t kUnknownDiskAvailability = -1ll;
 constexpr uint64_t kMegabyte = 1024ull * 1024;
+const int64_t kMinSecondsForPressureEvictions = 30;
 
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using MemoryAllocation = BlobMemoryController::MemoryAllocation;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
 using DiskSpaceFuncPtr = BlobMemoryController::DiskSpaceFuncPtr;
 
 // CrOS:
 // * Ram -  20%
 // * Disk - 50%
 //   Note: The disk is the user partition, so the operating system can still
@@ skipping 444 matching lines @@
 
 BlobMemoryController::BlobMemoryController(
     const base::FilePath& storage_directory,
     scoped_refptr<base::TaskRunner> file_runner)
     : file_paging_enabled_(file_runner.get() != nullptr),
       blob_storage_dir_(storage_directory),
       file_runner_(std::move(file_runner)),
       disk_space_function_(&base::SysInfo::AmountOfFreeDiskSpace),
       populated_memory_items_(
           base::MRUCache<uint64_t, ShareableBlobDataItem*>::NO_AUTO_EVICT),
+      memory_pressure_listener_(
+          base::Bind(&BlobMemoryController::OnMemoryPressure,
+                     base::Unretained(this))),
       weak_factory_(this) {}
 
 BlobMemoryController::~BlobMemoryController() {}
 
 void BlobMemoryController::DisableFilePaging(base::File::Error reason) {
   UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingDisabled", -reason,
                             -File::FILE_ERROR_MAX);
   DLOG(ERROR) << "Blob storage paging disabled, reason: " << reason;
   file_paging_enabled_ = false;
   in_flight_memory_used_ = 0;
@@ skipping 79 matching lines @@
   // more paging for any more pending blobs.
   if (!pending_memory_quota_tasks_.empty()) {
     return AppendMemoryTask(total_bytes_needed,
                             std::move(unreserved_memory_items), done_callback);
   }
 
   // Store right away if we can.
   if (total_bytes_needed <= GetAvailableMemoryForBlobs()) {
     GrantMemoryAllocations(&unreserved_memory_items,
                            static_cast<size_t>(total_bytes_needed));
-    MaybeScheduleEvictionUntilSystemHealthy();
+    MaybeScheduleEvictionUntilSystemHealthy(
+        base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
     done_callback.Run(true);
     return base::WeakPtr<QuotaAllocationTask>();
   }
 
   // Size is larger than available memory.
   DCHECK(pending_memory_quota_tasks_.empty());
   DCHECK_EQ(0u, pending_memory_quota_total_size_);
 
   auto weak_ptr = AppendMemoryTask(
       total_bytes_needed, std::move(unreserved_memory_items), done_callback);
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
   return weak_ptr;
 }
 
 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveFileQuota(
     std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
     const FileQuotaRequestCallback& done_callback) {
   pending_file_quota_tasks_.push_back(base::MakeUnique<FileQuotaAllocationTask>(
       this, disk_space_function_, std::move(unreserved_file_items),
       done_callback));
   pending_file_quota_tasks_.back()->set_my_list_position(
@@ skipping 13 matching lines @@
         items_paging_to_file_.end()) {
       return;
     }
     auto iterator = populated_memory_items_.Get(item->item_id());
     if (iterator == populated_memory_items_.end()) {
       populated_memory_items_bytes_ +=
           static_cast<size_t>(item->item()->length());
       populated_memory_items_.Put(item->item_id(), item.get());
     }
   }
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
 }
 
 void BlobMemoryController::CalculateBlobStorageLimits() {
   if (file_runner_) {
     PostTaskAndReplyWithResult(
         file_runner_.get(), FROM_HERE,
         base::Bind(&CalculateBlobStorageLimitsImpl, blob_storage_dir_, true),
         base::Bind(&BlobMemoryController::OnStorageLimitsCalculated,
                    weak_factory_.GetWeakPtr()));
   } else {
@@ skipping 101 matching lines @@
     std::unique_ptr<MemoryQuotaAllocationTask> memory_task =
         std::move(pending_memory_quota_tasks_.front());
     pending_memory_quota_tasks_.pop_front();
     pending_memory_quota_total_size_ -= memory_task->allocation_size();
     memory_task->RunDoneCallback(true);
   }
   RecordTracingCounters();
 }
 
 size_t BlobMemoryController::CollectItemsForEviction(
-    std::vector<scoped_refptr<ShareableBlobDataItem>>* output) {
+    std::vector<scoped_refptr<ShareableBlobDataItem>>* output,
+    uint64_t min_page_file_size) {
   base::CheckedNumeric<size_t> total_items_size = 0;
   // Process the recent item list and remove items until we have at least a
   // minimum file size or we're at the end of our items to page to disk.
-  while (total_items_size.ValueOrDie() < limits_.min_page_file_size &&
+  while (total_items_size.ValueOrDie() < min_page_file_size &&
          !populated_memory_items_.empty()) {
     auto iterator = --populated_memory_items_.end();
     ShareableBlobDataItem* item = iterator->second;
     DCHECK_EQ(item->item()->type(), DataElement::TYPE_BYTES);
     populated_memory_items_.Erase(iterator);
     size_t size = base::checked_cast<size_t>(item->item()->length());
     populated_memory_items_bytes_ -= size;
     total_items_size += size;
     output->push_back(make_scoped_refptr(item));
   }
   return total_items_size.ValueOrDie();
 }
 
-void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
+void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
   // Don't do eviction when others are happening, as we don't change our
   // pending_memory_quota_total_size_ value until after the paging files have
   // been written.
   if (pending_evictions_ != 0 || !file_paging_enabled_)
     return;
 
   uint64_t total_memory_usage =
       static_cast<uint64_t>(pending_memory_quota_total_size_) +
       blob_memory_used_;
 
+  size_t in_memory_limit = limits_.memory_limit_before_paging();
+  uint64_t min_page_file_size = limits_.min_page_file_size;
+  if (memory_pressure_level !=
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
+    in_memory_limit = 0;
+    // Use lower page file size to reduce using more memory for writing under
+    // pressure.
+    min_page_file_size = limits_.max_blob_in_memory_space *
+                         limits_.min_page_file_size_ratio_under_pressure;
+  }
+
   // We try to page items to disk until our current system size + requested
   // memory is below our size limit.
   // Size limit is a lower |memory_limit_before_paging()| if we have disk space.
   while (total_memory_usage > limits_.effective_max_disk_space ||
          (disk_used_ < limits_.effective_max_disk_space &&
-          total_memory_usage > limits_.memory_limit_before_paging())) {
+          total_memory_usage > in_memory_limit)) {
+    const char* reason = nullptr;
+    if (memory_pressure_level !=
+        base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
+      reason = "OnMemoryPressure";
+    } else if (total_memory_usage > limits_.effective_max_disk_space) {
+      reason = "SizeExceededMaxDiskSpace";
+    } else {
+      reason = "SizeExceededInMemoryLimit";
+    }
+
     // We only page when we have enough items to fill a whole page file.
-    if (populated_memory_items_bytes_ < limits_.min_page_file_size)
+    if (populated_memory_items_bytes_ < min_page_file_size)
       break;
-    DCHECK_LE(limits_.min_page_file_size,
-              static_cast<uint64_t>(blob_memory_used_));
+    DCHECK_LE(min_page_file_size, static_cast<uint64_t>(blob_memory_used_));
 
     std::vector<scoped_refptr<ShareableBlobDataItem>> items_to_swap;
-    size_t total_items_size = CollectItemsForEviction(&items_to_swap);
+
+    size_t total_items_size =
+        CollectItemsForEviction(&items_to_swap, min_page_file_size);
     if (total_items_size == 0)
       break;
 
     std::vector<DataElement*> items_for_paging;
     for (auto& shared_blob_item : items_to_swap) {
       items_paging_to_file_.insert(shared_blob_item->item_id());
       items_for_paging.push_back(shared_blob_item->item()->data_element_ptr());
     }
 
     // Update our bookkeeping.
@@ skipping 15 matching lines @@
 
     // Post the file writing task.
     base::PostTaskAndReplyWithResult(
         file_runner_.get(), FROM_HERE,
         base::Bind(&CreateFileAndWriteItems, blob_storage_dir_,
                    disk_space_function_, base::Passed(&page_file_path),
                    file_runner_, base::Passed(&items_for_paging),
                    total_items_size),
         base::Bind(&BlobMemoryController::OnEvictionComplete,
                    weak_factory_.GetWeakPtr(), base::Passed(&file_reference),
-                   base::Passed(&items_to_swap), total_items_size));
+                   base::Passed(&items_to_swap), total_items_size, reason,
+                   total_memory_usage));
+
+    last_eviction_time_ = base::TimeTicks::Now();
   }
+

[dmurph, 2017/05/10 19:13:34]

nit: remove newline

[ssid, 2017/05/12 01:38:01]

Done.

   RecordTracingCounters();
 }
 
 void BlobMemoryController::OnEvictionComplete(
     scoped_refptr<ShareableFileReference> file_reference,
     std::vector<scoped_refptr<ShareableBlobDataItem>> items,
     size_t total_items_size,
+    const char* evict_reason,
+    size_t memory_usage_before_eviction,
     std::pair<FileCreationInfo, int64_t /* avail_disk */> result) {
   if (!file_paging_enabled_)
     return;
 
   FileCreationInfo& file_info = std::get<0>(result);
   int64_t avail_disk_space = std::get<1>(result);
 
   if (file_info.error != File::FILE_OK) {
     DisableFilePaging(file_info.error);
     return;
@@ skipping 15 matching lines @@
         file_reference->path(), offset, shareable_item->item()->length(),
         file_info.last_modified);
     DCHECK(shareable_item->memory_allocation_);
     shareable_item->set_memory_allocation(nullptr);
     shareable_item->set_item(new_item);
     items_paging_to_file_.erase(shareable_item->item_id());
     offset += shareable_item->item()->length();
   }
   in_flight_memory_used_ -= total_items_size;
 
+  // Record change in memory usage at the last eviction reply.
+  size_t total_usage = blob_memory_used_ + pending_memory_quota_total_size_;
+  if (!pending_evictions_ && memory_usage_before_eviction >= total_usage) {
+    std::string full_histogram_name =
+        std::string("Storage.Blob.SizeEvictedToDiskInKB.") + evict_reason;
+    base::HistogramBase* histogram = base::Histogram::FactoryGet(
+        full_histogram_name, 1, 100000, 50,
+        base::HistogramBase::kUmaTargetedHistogramFlag);

[Ilya Sherman, 2017/05/11 20:13:39]

Please use one of the base::UmaHistogram... functions rather than calling
FactoryGet directly, unless you're trying to do something extra fancy here that
I'm missing.  (The functions are somewhat new, and now preferred over manual
FactoryGet invocations in most cases.)

[ssid, 2017/05/12 01:38:01]

Ah thanks! I couldn't find these functions when I searched. Fixed.

+    if (histogram)
+      histogram->Add((memory_usage_before_eviction - total_usage) / 1024);
+  }
+
   // We want callback on blobs up to the amount we've freed.
   MaybeGrantPendingMemoryRequests();
 
   // If we still have more blobs waiting and we're not waiting on more paging
   // operations, schedule more.
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
+}
+
+void BlobMemoryController::OnMemoryPressure(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
+  auto time_from_last_evicion = base::TimeTicks::Now() - last_eviction_time_;
+  if (time_from_last_evicion.InSeconds() < kMinSecondsForPressureEvictions)
+    return;
+
+  MaybeScheduleEvictionUntilSystemHealthy(memory_pressure_level);
 }
 
 FilePath BlobMemoryController::GenerateNextPageFileName() {
   std::string file_name = base::Uint64ToString(current_file_num_++);
   return blob_storage_dir_.Append(FilePath::FromUTF8Unsafe(file_name));
 }
 
 void BlobMemoryController::RecordTracingCounters() const {
   TRACE_COUNTER2("Blob", "MemoryUsage", "TotalStorage", blob_memory_used_,
                  "InFlightToDisk", in_flight_memory_used_);
@@ skipping 65 matching lines @@
   MaybeGrantPendingMemoryRequests();
 }
 
 void BlobMemoryController::OnBlobFileDelete(uint64_t size,
                                             const FilePath& path) {
   DCHECK_LE(size, disk_used_);
   disk_used_ -= size;
 }
 
 }  // namespace storage