Add memory pressure listener to Blob storage

storage/browser/blob/blob_memory_controller.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_memory_controller.h"
 
 #include <algorithm>
 #include <numeric>
 
 #include "base/bind.h"
@@ skipping 23 matching lines @@
 #include "storage/browser/blob/shareable_file_reference.h"
 #include "storage/common/data_element.h"
 
 using base::File;
 using base::FilePath;
 
 namespace storage {
 namespace {
 constexpr int64_t kUnknownDiskAvailability = -1ll;
 constexpr uint64_t kMegabyte = 1024ull * 1024;
+const int64_t kMinSecondsForPressureEvictions = 30;
 
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using MemoryAllocation = BlobMemoryController::MemoryAllocation;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
 using DiskSpaceFuncPtr = BlobMemoryController::DiskSpaceFuncPtr;
 
 // CrOS:
 // * Ram -  20%
 // * Disk - 50%
 //   Note: The disk is the user partition, so the operating system can still
@@ skipping 444 matching lines @@
 
 BlobMemoryController::BlobMemoryController(
     const base::FilePath& storage_directory,
     scoped_refptr<base::TaskRunner> file_runner)
     : file_paging_enabled_(file_runner.get() != nullptr),
       blob_storage_dir_(storage_directory),
       file_runner_(std::move(file_runner)),
       disk_space_function_(&base::SysInfo::AmountOfFreeDiskSpace),
       populated_memory_items_(
           base::MRUCache<uint64_t, ShareableBlobDataItem*>::NO_AUTO_EVICT),
+      memory_pressure_listener_(
+          base::Bind(&BlobMemoryController::OnMemoryPressure,
+                     base::Unretained(this))),

[dmurph, 2017/05/09 21:57:33]

Unretained? How is this being registered? Can we use weak ptrs?

[ssid, 2017/05/10 00:21:38]

The unregistration is safe because the callback is only invoked on the correct
thread, and the notification is also always invoked on the same thread. The code
here prevents the observer from getting accessed after unregistration:
https://chromium.googlesource.com/chromium/src/+/a39b866eac82e716d65def1bd9a4...
I did not add it because it is not required.

Also just realized that i cannot add GetWeakPtr() here since the weak_factory_
is initialized later. I cannot move the weak_factory_ up in the initialization
because of static assert that requires the weak_factory_ to always be the last
member in the class.

Do you think I should make this unique_ptr and pass weak ptr? Or I could add a
comment to explain why unretained.

[dmurph, 2017/05/10 19:13:34]

If you think that it won't get called after-free, then that's fine as long as
you comment how that invariant is kept. I'm assuming this is done by how the
listener is registered, which isn't in this patch, correct?

(you could also use a weakptr and just create the callback in the constructor
method)
{
  memory_pressure_listener_ =
base::Bind(&BlobMemoryController::OnMemoryPressure, weak_factory_.GetWeakPtr())
}

But then it can't be const.

       weak_factory_(this) {}
 
 BlobMemoryController::~BlobMemoryController() {}
 
 void BlobMemoryController::DisableFilePaging(base::File::Error reason) {
   UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingDisabled", -reason,
                             -File::FILE_ERROR_MAX);
   DLOG(ERROR) << "Blob storage paging disabled, reason: " << reason;
   file_paging_enabled_ = false;
   in_flight_memory_used_ = 0;
@@ skipping 79 matching lines @@
   // more paging for any more pending blobs.
   if (!pending_memory_quota_tasks_.empty()) {
     return AppendMemoryTask(total_bytes_needed,
                             std::move(unreserved_memory_items), done_callback);
   }
 
   // Store right away if we can.
   if (total_bytes_needed <= GetAvailableMemoryForBlobs()) {
     GrantMemoryAllocations(&unreserved_memory_items,
                            static_cast<size_t>(total_bytes_needed));
-    MaybeScheduleEvictionUntilSystemHealthy();
+    MaybeScheduleEvictionUntilSystemHealthy(false /* on_memory_pressue */);
     done_callback.Run(true);
     return base::WeakPtr<QuotaAllocationTask>();
   }
 
   // Size is larger than available memory.
   DCHECK(pending_memory_quota_tasks_.empty());
   DCHECK_EQ(0u, pending_memory_quota_total_size_);
 
   auto weak_ptr = AppendMemoryTask(
       total_bytes_needed, std::move(unreserved_memory_items), done_callback);
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(false /* on_memory_pressue */);
   return weak_ptr;
 }
 
 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveFileQuota(
     std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
     const FileQuotaRequestCallback& done_callback) {
   pending_file_quota_tasks_.push_back(base::MakeUnique<FileQuotaAllocationTask>(
       this, disk_space_function_, std::move(unreserved_file_items),
       done_callback));
   pending_file_quota_tasks_.back()->set_my_list_position(
@@ skipping 13 matching lines @@
         items_paging_to_file_.end()) {
       return;
     }
     auto iterator = populated_memory_items_.Get(item->item_id());
     if (iterator == populated_memory_items_.end()) {
       populated_memory_items_bytes_ +=
           static_cast<size_t>(item->item()->length());
       populated_memory_items_.Put(item->item_id(), item.get());
     }
   }
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(false /* on_memory_pressue */);
 }
 
 void BlobMemoryController::CalculateBlobStorageLimits() {
   if (file_runner_) {
     PostTaskAndReplyWithResult(
         file_runner_.get(), FROM_HERE,
         base::Bind(&CalculateBlobStorageLimitsImpl, blob_storage_dir_, true),
         base::Bind(&BlobMemoryController::OnStorageLimitsCalculated,
                    weak_factory_.GetWeakPtr()));
   } else {
@@ skipping 101 matching lines @@
     std::unique_ptr<MemoryQuotaAllocationTask> memory_task =
         std::move(pending_memory_quota_tasks_.front());
     pending_memory_quota_tasks_.pop_front();
     pending_memory_quota_total_size_ -= memory_task->allocation_size();
     memory_task->RunDoneCallback(true);
   }
   RecordTracingCounters();
 }
 
 size_t BlobMemoryController::CollectItemsForEviction(
-    std::vector<scoped_refptr<ShareableBlobDataItem>>* output) {
+    std::vector<scoped_refptr<ShareableBlobDataItem>>* output,
+    uint64_t min_page_file_size) {
   base::CheckedNumeric<size_t> total_items_size = 0;
   // Process the recent item list and remove items until we have at least a
   // minimum file size or we're at the end of our items to page to disk.
-  while (total_items_size.ValueOrDie() < limits_.min_page_file_size &&
+  while (total_items_size.ValueOrDie() < min_page_file_size &&
          !populated_memory_items_.empty()) {
     auto iterator = --populated_memory_items_.end();
     ShareableBlobDataItem* item = iterator->second;
     DCHECK_EQ(item->item()->type(), DataElement::TYPE_BYTES);
     populated_memory_items_.Erase(iterator);
     size_t size = base::checked_cast<size_t>(item->item()->length());
     populated_memory_items_bytes_ -= size;
     total_items_size += size;
     output->push_back(make_scoped_refptr(item));
   }
   return total_items_size.ValueOrDie();
 }
 
-void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
+void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy(
+    bool on_memory_pressure) {
   // Don't do eviction when others are happening, as we don't change our
   // pending_memory_quota_total_size_ value until after the paging files have
   // been written.
   if (pending_evictions_ != 0 || !file_paging_enabled_)
     return;
 
   uint64_t total_memory_usage =
       static_cast<uint64_t>(pending_memory_quota_total_size_) +
       blob_memory_used_;
 
+  size_t in_memory_limit = limits_.memory_limit_before_paging();
+  uint64_t min_page_file_size = limits_.min_page_file_size;
+  if (on_memory_pressure) {
+    in_memory_limit = 0;

[dmurph, 2017/05/09 21:57:33]

Do we want to flush everything?

[ssid, 2017/05/10 00:21:38]

Actually we would still not flush anything less than
min_page_file_size_under_pressure. So, that number tells maximum size keep in
memory, right?

+    // Use lower page file size to reduce using more memory for writing under
+    // pressure.
+    min_page_file_size = limits_.min_page_file_size_under_pressure;
+  }
+
   // We try to page items to disk until our current system size + requested
   // memory is below our size limit.
   // Size limit is a lower |memory_limit_before_paging()| if we have disk space.
   while (total_memory_usage > limits_.effective_max_disk_space ||
          (disk_used_ < limits_.effective_max_disk_space &&
-          total_memory_usage > limits_.memory_limit_before_paging())) {
+          total_memory_usage > in_memory_limit)) {
+    const char* reason = nullptr;
+    if (on_memory_pressure) {
+      reason = "OnMemoryPressure";
+    } else if (total_memory_usage > limits_.effective_max_disk_space) {
+      reason = "SizeExceededMaxDiskSpace";
+    } else {
+      reason = "SizeExceededInMemoryLimit";
+    }
+
     // We only page when we have enough items to fill a whole page file.
-    if (populated_memory_items_bytes_ < limits_.min_page_file_size)
+    if (populated_memory_items_bytes_ < min_page_file_size)
       break;
-    DCHECK_LE(limits_.min_page_file_size,
-              static_cast<uint64_t>(blob_memory_used_));
+    DCHECK_LE(min_page_file_size, static_cast<uint64_t>(blob_memory_used_));
 
     std::vector<scoped_refptr<ShareableBlobDataItem>> items_to_swap;
-    size_t total_items_size = CollectItemsForEviction(&items_to_swap);
+
+    size_t total_items_size =
+        CollectItemsForEviction(&items_to_swap, min_page_file_size);
     if (total_items_size == 0)
       break;
 
     std::vector<DataElement*> items_for_paging;
     for (auto& shared_blob_item : items_to_swap) {
       items_paging_to_file_.insert(shared_blob_item->item_id());
       items_for_paging.push_back(shared_blob_item->item()->data_element_ptr());
     }
 
     // Update our bookkeeping.
@@ skipping 15 matching lines @@
 
     // Post the file writing task.
     base::PostTaskAndReplyWithResult(
         file_runner_.get(), FROM_HERE,
         base::Bind(&CreateFileAndWriteItems, blob_storage_dir_,
                    disk_space_function_, base::Passed(&page_file_path),
                    file_runner_, base::Passed(&items_for_paging),
                    total_items_size),
         base::Bind(&BlobMemoryController::OnEvictionComplete,
                    weak_factory_.GetWeakPtr(), base::Passed(&file_reference),
-                   base::Passed(&items_to_swap), total_items_size));
+                   base::Passed(&items_to_swap), total_items_size, reason,
+                   total_memory_usage));
+
+    last_eviction_time_ = base::TimeTicks::Now();
   }
+
   RecordTracingCounters();
 }
 
 void BlobMemoryController::OnEvictionComplete(
     scoped_refptr<ShareableFileReference> file_reference,
     std::vector<scoped_refptr<ShareableBlobDataItem>> items,
     size_t total_items_size,
+    const char* evict_reason,
+    size_t memory_usage_before_eviction,
     std::pair<FileCreationInfo, int64_t /* avail_disk */> result) {
   if (!file_paging_enabled_)
     return;
 
   FileCreationInfo& file_info = std::get<0>(result);
   int64_t avail_disk_space = std::get<1>(result);
 
   if (file_info.error != File::FILE_OK) {
     DisableFilePaging(file_info.error);
     return;
@@ skipping 15 matching lines @@
         file_reference->path(), offset, shareable_item->item()->length(),
         file_info.last_modified);
     DCHECK(shareable_item->memory_allocation_);
     shareable_item->set_memory_allocation(nullptr);
     shareable_item->set_item(new_item);
     items_paging_to_file_.erase(shareable_item->item_id());
     offset += shareable_item->item()->length();
   }
   in_flight_memory_used_ -= total_items_size;
 
+  // Record change in memory usage at the last eviction reply.
+  size_t total_usage = blob_memory_used_ + pending_memory_quota_total_size_;
+  if (!pending_evictions_ && memory_usage_before_eviction >= total_usage) {
+    std::string full_histogram_name =
+        std::string("Storage.Blob.SizeEvictedToDiskInKB.") + evict_reason;
+    base::HistogramBase* histogram = base::Histogram::FactoryGet(
+        full_histogram_name, 1, 100000, 50,
+        base::HistogramBase::kUmaTargetedHistogramFlag);
+    if (histogram)
+      histogram->Add((memory_usage_before_eviction - total_usage) / 1024);
+  }
+
   // We want callback on blobs up to the amount we've freed.
   MaybeGrantPendingMemoryRequests();
 
   // If we still have more blobs waiting and we're not waiting on more paging
   // operations, schedule more.
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(false /* on_memory_pressue */);
+}
+
+void BlobMemoryController::OnMemoryPressure(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
+  auto time_from_last_evicion = base::TimeTicks::Now() - last_eviction_time_;
+  if (time_from_last_evicion.InSeconds() < kMinSecondsForPressureEvictions)
+    return;
+
+  MaybeScheduleEvictionUntilSystemHealthy(true /* on_memory_pressure */);
 }
 
 FilePath BlobMemoryController::GenerateNextPageFileName() {
   std::string file_name = base::Uint64ToString(current_file_num_++);
   return blob_storage_dir_.Append(FilePath::FromUTF8Unsafe(file_name));
 }
 
 void BlobMemoryController::RecordTracingCounters() const {
   TRACE_COUNTER2("Blob", "MemoryUsage", "TotalStorage", blob_memory_used_,
                  "InFlightToDisk", in_flight_memory_used_);
@@ skipping 65 matching lines @@
   MaybeGrantPendingMemoryRequests();
 }
 
 void BlobMemoryController::OnBlobFileDelete(uint64_t size,
                                             const FilePath& path) {
   DCHECK_LE(size, disk_used_);
   disk_used_ -= size;
 }
 
 }  // namespace storage