[BlobStorage] Enabling disk paging and direct storage.

storage/browser/blob/blob_memory_controller.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_memory_controller.h"
 
 #include <algorithm>
 #include <numeric>
 
+#include "base/bind.h"
+#include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/callback_helpers.h"
 #include "base/containers/small_map.h"
 #include "base/files/file_util.h"
 #include "base/guid.h"
 #include "base/location.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/numerics/safe_math.h"
 #include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
+#include "base/sys_info.h"
 #include "base/task_runner.h"
 #include "base/task_runner_util.h"
 #include "base/threading/thread_restrictions.h"
 #include "base/time/time.h"
 #include "base/trace_event/trace_event.h"
-#include "base/tuple.h"
 #include "storage/browser/blob/blob_data_builder.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
 #include "storage/browser/blob/shareable_file_reference.h"
 #include "storage/common/data_element.h"
 
 using base::File;
 using base::FilePath;
 
 namespace storage {
 namespace {
+constexpr int64_t kUnknownDiskAvailability = -1ll;
+constexpr uint64_t kMegabyte = 1024ull * 1024;
+
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using MemoryAllocation = BlobMemoryController::MemoryAllocation;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+using DiskSpaceFuncPtr = BlobMemoryController::DiskSpaceFuncPtr;
+
+// CrOS:
+// * Ram -  20%
+// * Disk - 50%
+//   Note: The disk is the user partition, so the operating system can still
+//   function if this is full.
+// Android:
+// * RAM -  20%
+// * Disk -  5%
+// Desktop:
+// * Ram -  20%, or 2 GB if x64.
+// * Disk - 10%
+BlobStorageLimits CalculateBlobStorageLimitsImpl(const FilePath& storage_dir,
+                                                 bool disk_enabled) {
+  int64_t disk_size =
+      disk_enabled ? base::SysInfo::AmountOfTotalDiskSpace(storage_dir) : 0ull;
+  int64_t memory_size = base::SysInfo::AmountOfPhysicalMemory();
+
+  BlobStorageLimits limits;
+
+  // Don't do specialty configuration for error size (-1).
+  if (memory_size > 0) {
+#if !defined(OS_CHROMEOS) && !defined(OS_ANDROID) && defined(ARCH_CPU_64_BITS)
+    constexpr size_t kTwoGigabytes = 2ull * 1024 * 1024 * 1024;
+    limits.max_blob_in_memory_space = kTwoGigabytes;
+#else
+    limits.max_blob_in_memory_space = static_cast<size_t>(memory_size / 5ll);
+#endif
+  }
+
+  // Don't do specialty configuration for error size (-1). Allow no disk.
+  if (disk_size >= 0) {
+#if defined(OS_CHROMEOS)
+    limits.desired_max_disk_space = static_cast<uint64_t>(disk_size / 2ll);
+#elif defined(OS_ANDROID)
+    limits.desired_max_disk_space = static_cast<uint64_t>(disk_size / 20ll);
+#else
+    limits.desired_max_disk_space = static_cast<uint64_t>(disk_size / 10ll);
+#endif
+  }
+  UMA_HISTOGRAM_COUNTS_1M("Storage.Blob.MaxDiskSpace",
+                          limits.desired_max_disk_space / kMegabyte);
+  limits.effective_max_disk_space = limits.desired_max_disk_space;
+
+  return limits;
+}
 
 File::Error CreateBlobDirectory(const FilePath& blob_storage_dir) {
   File::Error error = File::FILE_OK;
   base::CreateDirectoryAndGetError(blob_storage_dir, &error);
   UMA_HISTOGRAM_ENUMERATION("Storage.Blob.CreateDirectoryResult", -error,
                             -File::FILE_ERROR_MAX);
   DLOG_IF(ERROR, error != File::FILE_OK)
       << "Error creating blob storage directory: " << error;
   return error;
 }
 
 void DestructFile(File infos_without_references) {}
 
+void DeleteFiles(std::vector<FileCreationInfo> files) {
+  for (FileCreationInfo& file_info : files) {
+    file_info.file.Close();
+    base::DeleteFile(file_info.path, false);
+  }
+}
+
+struct EmptyFilesResult {
+  EmptyFilesResult() {}
+  EmptyFilesResult(std::vector<FileCreationInfo> files,
+                   File::Error file_error,
+                   int64_t disk_availability)
+      : files(std::move(files)),
+        file_error(file_error),
+        disk_availability(disk_availability) {}
+  ~EmptyFilesResult() {}
+  EmptyFilesResult(EmptyFilesResult&& o) = default;
+  EmptyFilesResult& operator=(EmptyFilesResult&& other) = default;
+
+  std::vector<FileCreationInfo> files;
+  File::Error file_error = File::FILE_ERROR_FAILED;
+  int64_t disk_availability = 0;
+};
+
 // Used for new unpopulated file items. Caller must populate file reference in
-// returned FileCreationInfos.
-std::pair<std::vector<FileCreationInfo>, File::Error> CreateEmptyFiles(
+// returned FileCreationInfos. Also returns the currently available disk space
+// (without the future size of these files).
+EmptyFilesResult CreateEmptyFiles(
     const FilePath& blob_storage_dir,
+    DiskSpaceFuncPtr disk_space_function,
     scoped_refptr<base::TaskRunner> file_task_runner,
     std::vector<base::FilePath> file_paths) {
   base::ThreadRestrictions::AssertIOAllowed();
 
   File::Error dir_create_status = CreateBlobDirectory(blob_storage_dir);
-  if (dir_create_status != File::FILE_OK)
-    return std::make_pair(std::vector<FileCreationInfo>(), dir_create_status);
+  if (dir_create_status != File::FILE_OK) {
+    return EmptyFilesResult(std::vector<FileCreationInfo>(), dir_create_status,
+                            kUnknownDiskAvailability);
+  }
+
+  int64_t free_disk_space = disk_space_function(blob_storage_dir);
 
   std::vector<FileCreationInfo> result;
   for (const base::FilePath& file_path : file_paths) {
     FileCreationInfo creation_info;
     // Try to open our file.
     File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
     creation_info.path = std::move(file_path);
     creation_info.file_deletion_runner = file_task_runner;
     creation_info.error = file.error_details();
     if (creation_info.error != File::FILE_OK) {
-      return std::make_pair(std::vector<FileCreationInfo>(),
-                            creation_info.error);
+      return EmptyFilesResult(std::vector<FileCreationInfo>(),
+                              creation_info.error, free_disk_space);
     }
     creation_info.file = std::move(file);
 
     result.push_back(std::move(creation_info));
   }
-  return std::make_pair(std::move(result), File::FILE_OK);
+  return EmptyFilesResult(std::move(result), File::FILE_OK, free_disk_space);
 }
 
 // Used to evict multiple memory items out to a single file. Caller must
-// populate file reference in returned FileCreationInfo.
-FileCreationInfo CreateFileAndWriteItems(
+// populate file reference in returned FileCreationInfo. Also returns the free
+// disk space AFTER creating this file.
+std::pair<FileCreationInfo, int64_t> CreateFileAndWriteItems(
     const FilePath& blob_storage_dir,
+    DiskSpaceFuncPtr disk_space_function,
     const FilePath& file_path,
     scoped_refptr<base::TaskRunner> file_task_runner,
     std::vector<DataElement*> items,
     size_t total_size_bytes) {
   DCHECK_NE(0u, total_size_bytes);
   UMA_HISTOGRAM_MEMORY_KB("Storage.Blob.PageFileSize", total_size_bytes / 1024);
   base::ThreadRestrictions::AssertIOAllowed();
 
   FileCreationInfo creation_info;
   creation_info.file_deletion_runner = std::move(file_task_runner);
   creation_info.error = CreateBlobDirectory(blob_storage_dir);
   if (creation_info.error != File::FILE_OK)
-    return creation_info;
+    return std::make_pair(std::move(creation_info), kUnknownDiskAvailability);
+
+  int64_t free_disk_space = disk_space_function(blob_storage_dir);
+
+  // Fail early instead of creating the files if we fill the disk.
+  if (free_disk_space != kUnknownDiskAvailability &&
+      free_disk_space < static_cast<int64_t>(total_size_bytes)) {
+    creation_info.error = File::FILE_ERROR_NO_SPACE;
+    return std::make_pair(std::move(creation_info), free_disk_space);
+  }
+  int64_t disk_availability =
+      free_disk_space == kUnknownDiskAvailability
+          ? kUnknownDiskAvailability
+          : free_disk_space - static_cast<int64_t>(total_size_bytes);
 
   // Create the page file.
   File file(file_path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
   creation_info.path = file_path;
   creation_info.error = file.error_details();
   if (creation_info.error != File::FILE_OK)
-    return creation_info;
+    return std::make_pair(std::move(creation_info), free_disk_space);
 
   // Write data.
   file.SetLength(total_size_bytes);
   int bytes_written = 0;
   for (DataElement* element : items) {
     DCHECK_EQ(DataElement::TYPE_BYTES, element->type());
     size_t length = base::checked_cast<size_t>(element->length());
     size_t bytes_left = length;
     while (bytes_left > 0) {
       bytes_written =
           file.WriteAtCurrentPos(element->bytes() + (length - bytes_left),
                                  base::saturated_cast<int>(bytes_left));
       if (bytes_written < 0)
         break;
       DCHECK_LE(static_cast<size_t>(bytes_written), bytes_left);
       bytes_left -= bytes_written;
     }
     if (bytes_written < 0)
       break;
   }
   if (!file.Flush()) {
+    file.Close();
+    base::DeleteFile(file_path, false);
     creation_info.error = File::FILE_ERROR_FAILED;
-    return creation_info;
+    return std::make_pair(std::move(creation_info), free_disk_space);
   }
 
   File::Info info;
   bool success = file.GetInfo(&info);
   creation_info.error =
       bytes_written < 0 || !success ? File::FILE_ERROR_FAILED : File::FILE_OK;
   creation_info.last_modified = info.last_modified;
-  return creation_info;
+  return std::make_pair(std::move(creation_info), disk_availability);
 }
 
 uint64_t GetTotalSizeAndFileSizes(
     const std::vector<scoped_refptr<ShareableBlobDataItem>>&
         unreserved_file_items,
     std::vector<uint64_t>* file_sizes_output) {
   uint64_t total_size_output = 0;
   base::SmallMap<std::map<uint64_t, uint64_t>> file_id_to_sizes;
   for (const auto& item : unreserved_file_items) {
     const DataElement& element = item->item()->data_element();
@@ skipping 96 matching lines @@
   base::WeakPtrFactory<MemoryQuotaAllocationTask> weak_factory_;
   DISALLOW_COPY_AND_ASSIGN(MemoryQuotaAllocationTask);
 };
 
 class BlobMemoryController::FileQuotaAllocationTask
     : public BlobMemoryController::QuotaAllocationTask {
  public:
   // We post a task to create the file for the items right away.
   FileQuotaAllocationTask(
       BlobMemoryController* memory_controller,
+      DiskSpaceFuncPtr disk_space_function,
       std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
       const FileQuotaRequestCallback& done_callback)
       : controller_(memory_controller),
         done_callback_(done_callback),
         weak_factory_(this) {
     // Get the file sizes and total size.
-    std::vector<uint64_t> file_sizes;
     uint64_t total_size =
-        GetTotalSizeAndFileSizes(unreserved_file_items, &file_sizes);
+        GetTotalSizeAndFileSizes(unreserved_file_items, &file_sizes_);
     DCHECK_LE(total_size, controller_->GetAvailableFileSpaceForBlobs());
     allocation_size_ = total_size;
 
     // Check & set our item states.
     for (auto& shareable_item : unreserved_file_items) {
       DCHECK_EQ(ShareableBlobDataItem::QUOTA_NEEDED, shareable_item->state());
       DCHECK_EQ(DataElement::TYPE_FILE, shareable_item->item()->type());
       shareable_item->set_state(ShareableBlobDataItem::QUOTA_REQUESTED);
     }
     pending_items_ = std::move(unreserved_file_items);
 
     // Increment disk usage and create our file references.
     controller_->disk_used_ += allocation_size_;
     std::vector<base::FilePath> file_paths;
     std::vector<scoped_refptr<ShareableFileReference>> references;
-    for (size_t i = 0; i < file_sizes.size(); i++) {
+    for (size_t i = 0; i < file_sizes_.size(); i++) {
       file_paths.push_back(controller_->GenerateNextPageFileName());
       references.push_back(ShareableFileReference::GetOrCreate(
           file_paths.back(), ShareableFileReference::DELETE_ON_FINAL_RELEASE,
           controller_->file_runner_.get()));
-      references.back()->AddFinalReleaseCallback(
-          base::Bind(&BlobMemoryController::OnBlobFileDelete,
-                     controller_->weak_factory_.GetWeakPtr(), file_sizes[i]));
     }
-
     // Send file creation task to file thread.
     base::PostTaskAndReplyWithResult(
         controller_->file_runner_.get(), FROM_HERE,
         base::Bind(&CreateEmptyFiles, controller_->blob_storage_dir_,
-                   controller_->file_runner_, base::Passed(&file_paths)),
+                   disk_space_function, controller_->file_runner_,
+                   base::Passed(&file_paths)),
         base::Bind(&FileQuotaAllocationTask::OnCreateEmptyFiles,
-                   weak_factory_.GetWeakPtr(), base::Passed(&references)));
+                   weak_factory_.GetWeakPtr(), base::Passed(&references),
+                   allocation_size_));
     controller_->RecordTracingCounters();
   }
   ~FileQuotaAllocationTask() override {}
 
   void RunDoneCallback(std::vector<FileCreationInfo> file_info, bool success) {
     // Make sure we clear the weak pointers we gave to the caller beforehand.
     weak_factory_.InvalidateWeakPtrs();
 
     // We want to destroy this object on the exit of this method if we were
     // successful.
     std::unique_ptr<FileQuotaAllocationTask> this_object;
     if (success) {
+      // Register the disk space accounting callback.
+      DCHECK_EQ(file_info.size(), file_sizes_.size());
+      for (size_t i = 0; i < file_sizes_.size(); i++) {
+        file_info[i].file_reference->AddFinalReleaseCallback(base::Bind(
+            &BlobMemoryController::OnBlobFileDelete,
+            controller_->weak_factory_.GetWeakPtr(), file_sizes_[i]));
+      }
       for (auto& item : pending_items_) {
         item->set_state(ShareableBlobDataItem::QUOTA_GRANTED);
       }
       this_object = std::move(*my_list_position_);
       controller_->pending_file_quota_tasks_.erase(my_list_position_);
     }
 
     done_callback_.Run(std::move(file_info), success);
   }
 
   base::WeakPtr<QuotaAllocationTask> GetWeakPtr() {
     return weak_factory_.GetWeakPtr();
   }
 
   void Cancel() override {
-    // This call destroys this object. We rely on ShareableFileReference's
-    // final release callback for disk_usage_ accounting.
+    DCHECK_GE(controller_->disk_used_, allocation_size_);
+    controller_->disk_used_ -= allocation_size_;
+    // This call destroys this object.
     controller_->pending_file_quota_tasks_.erase(my_list_position_);
   }
 
   void OnCreateEmptyFiles(
       std::vector<scoped_refptr<ShareableFileReference>> references,
-      std::pair<std::vector<FileCreationInfo>, File::Error> files_and_error) {
-    auto& files = files_and_error.first;
-    if (files.empty()) {
+      uint64_t new_files_total_size,
+      EmptyFilesResult result) {
+    int64_t avail_disk_space = result.disk_availability;
+    if (result.files.empty()) {
+      DCHECK_NE(result.file_error, File::FILE_OK);
       DCHECK_GE(controller_->disk_used_, allocation_size_);
       controller_->disk_used_ -= allocation_size_;
       // This will call our callback and delete the object correctly.
-      controller_->DisableFilePaging(files_and_error.second);
+      controller_->DisableFilePaging(result.file_error);
       return;
     }
-    DCHECK_EQ(files.size(), references.size());
-    for (size_t i = 0; i < files.size(); i++) {
-      files[i].file_reference = std::move(references[i]);
+    // The allocation won't fit at all. Cancel this request. The disk will be
+    // decremented when the file is deleted through AddFinalReleaseCallback.
+    if (avail_disk_space != kUnknownDiskAvailability &&
+        base::checked_cast<uint64_t>(avail_disk_space) < new_files_total_size) {
+      DCHECK_GE(controller_->disk_used_, allocation_size_);
+      controller_->disk_used_ -= allocation_size_;
+      controller_->AdjustDiskUsage(static_cast<uint64_t>(avail_disk_space));
+      controller_->file_runner_->PostTask(
+          FROM_HERE, base::Bind(&DeleteFiles, base::Passed(&result.files)));
+      std::unique_ptr<FileQuotaAllocationTask> this_object =
+          std::move(*my_list_position_);
+      controller_->pending_file_quota_tasks_.erase(my_list_position_);
+      RunDoneCallback(std::vector<FileCreationInfo>(), false);
+      return;
     }
-    RunDoneCallback(std::move(files), true);
+    if (avail_disk_space != kUnknownDiskAvailability) {
+      controller_->AdjustDiskUsage(base::checked_cast<uint64_t>(
+          avail_disk_space - new_files_total_size));
+    }
+    DCHECK_EQ(result.files.size(), references.size());
+    for (size_t i = 0; i < result.files.size(); i++) {
+      result.files[i].file_reference = std::move(references[i]);
+    }
+    RunDoneCallback(std::move(result.files), true);
   }
 
   // The my_list_position_ iterator is stored so that we can remove ourself
   // from the task list when we are cancelled.
   void set_my_list_position(
       PendingFileQuotaTaskList::iterator my_list_position) {
     my_list_position_ = my_list_position;
   }
 
+  size_t allocation_size() const { return allocation_size_; }
+
  private:
   BlobMemoryController* controller_;
+  std::vector<uint64_t> file_sizes_;
   std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items_;
-  scoped_refptr<base::TaskRunner> file_runner_;
   FileQuotaRequestCallback done_callback_;
 
   uint64_t allocation_size_;
   PendingFileQuotaTaskList::iterator my_list_position_;
 
   base::WeakPtrFactory<FileQuotaAllocationTask> weak_factory_;
   DISALLOW_COPY_AND_ASSIGN(FileQuotaAllocationTask);
 };
 
 BlobMemoryController::BlobMemoryController(
     const base::FilePath& storage_directory,
     scoped_refptr<base::TaskRunner> file_runner)
     : file_paging_enabled_(file_runner.get() != nullptr),
       blob_storage_dir_(storage_directory),
       file_runner_(std::move(file_runner)),
+      disk_space_function_(&base::SysInfo::AmountOfFreeDiskSpace),
       populated_memory_items_(
           base::MRUCache<uint64_t, ShareableBlobDataItem*>::NO_AUTO_EVICT),
       weak_factory_(this) {}
 
 BlobMemoryController::~BlobMemoryController() {}
 
 void BlobMemoryController::DisableFilePaging(base::File::Error reason) {
   UMA_HISTOGRAM_ENUMERATION("Storage.Blob.PagingDisabled", -reason,
                             -File::FILE_ERROR_MAX);
   DLOG(ERROR) << "Blob storage paging disabled, reason: " << reason;
   file_paging_enabled_ = false;
   in_flight_memory_used_ = 0;
   items_paging_to_file_.clear();
   pending_evictions_ = 0;
   pending_memory_quota_total_size_ = 0;
   populated_memory_items_.Clear();
   populated_memory_items_bytes_ = 0;
   file_runner_ = nullptr;
 
   PendingMemoryQuotaTaskList old_memory_tasks;
   PendingFileQuotaTaskList old_file_tasks;
   std::swap(old_memory_tasks, pending_memory_quota_tasks_);
   std::swap(old_file_tasks, pending_file_quota_tasks_);
 
   // Don't call the callbacks until we have a consistent state.
   for (auto& memory_request : old_memory_tasks) {
     memory_request->RunDoneCallback(false);
   }
   for (auto& file_request : old_file_tasks) {
+    // OnBlobFileDelete is registered when RunDoneCallback is called with
+    // |true|, so manually do disk accounting.
+    disk_used_ -= file_request->allocation_size();
     file_request->RunDoneCallback(std::vector<FileCreationInfo>(), false);
   }
 }
 
 BlobMemoryController::Strategy BlobMemoryController::DetermineStrategy(
     size_t preemptive_transported_bytes,
     uint64_t total_transportation_bytes) const {
   if (total_transportation_bytes == 0)
     return Strategy::NONE_NEEDED;
   if (!CanReserveQuota(total_transportation_bytes))
     return Strategy::TOO_LARGE;
 
   // Handle the case where we have all the bytes preemptively transported, and
   // we can also fit them.
   if (preemptive_transported_bytes == total_transportation_bytes &&
       pending_memory_quota_tasks_.empty() &&
-      preemptive_transported_bytes < GetAvailableMemoryForBlobs()) {
+      preemptive_transported_bytes <= GetAvailableMemoryForBlobs()) {
     return Strategy::NONE_NEEDED;
   }
   if (file_paging_enabled_ &&
-      (total_transportation_bytes > limits_.memory_limit_before_paging())) {
+      total_transportation_bytes <= GetAvailableFileSpaceForBlobs() &&
+      total_transportation_bytes > limits_.memory_limit_before_paging()) {
     return Strategy::FILE;
   }
   if (total_transportation_bytes > limits_.max_ipc_memory_size)
     return Strategy::SHARED_MEMORY;
   return Strategy::IPC;
 }
 
 bool BlobMemoryController::CanReserveQuota(uint64_t size) const {
   // We check each size independently as a blob can't be constructed in both
   // disk and memory.
@@ skipping 46 matching lines @@
   auto weak_ptr = AppendMemoryTask(
       total_bytes_needed, std::move(unreserved_memory_items), done_callback);
   MaybeScheduleEvictionUntilSystemHealthy();
   return weak_ptr;
 }
 
 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveFileQuota(
     std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
     const FileQuotaRequestCallback& done_callback) {
   pending_file_quota_tasks_.push_back(base::MakeUnique<FileQuotaAllocationTask>(
-      this, std::move(unreserved_file_items), done_callback));
+      this, disk_space_function_, std::move(unreserved_file_items),
+      done_callback));
   pending_file_quota_tasks_.back()->set_my_list_position(
       --pending_file_quota_tasks_.end());
   return pending_file_quota_tasks_.back()->GetWeakPtr();
 }
 
 void BlobMemoryController::NotifyMemoryItemsUsed(
     const std::vector<scoped_refptr<ShareableBlobDataItem>>& items) {
   for (const auto& item : items) {
     if (item->item()->type() != DataElement::TYPE_BYTES ||
         item->state() != ShareableBlobDataItem::POPULATED_WITH_QUOTA) {
       continue;
     }
     // We don't want to re-add the item if we're currently paging it to disk.
     if (items_paging_to_file_.find(item->item_id()) !=
         items_paging_to_file_.end()) {
       return;
     }
     auto iterator = populated_memory_items_.Get(item->item_id());
     if (iterator == populated_memory_items_.end()) {
       populated_memory_items_bytes_ +=
           static_cast<size_t>(item->item()->length());
       populated_memory_items_.Put(item->item_id(), item.get());
     }
   }
   MaybeScheduleEvictionUntilSystemHealthy();
 }
 
+void BlobMemoryController::CalculateBlobStorageLimits() {
+  if (file_runner_) {
+    PostTaskAndReplyWithResult(
+        file_runner_.get(), FROM_HERE,
+        base::Bind(&CalculateBlobStorageLimitsImpl, blob_storage_dir_, true),
+        base::Bind(&BlobMemoryController::OnStorageLimitsCalculated,
+                   weak_factory_.GetWeakPtr()));
+  } else {
+    OnStorageLimitsCalculated(
+        CalculateBlobStorageLimitsImpl(blob_storage_dir_, false));
+  }
+}
+
+base::WeakPtr<BlobMemoryController> BlobMemoryController::GetWeakPtr() {
+  return weak_factory_.GetWeakPtr();
+}
+
+void BlobMemoryController::OnStorageLimitsCalculated(BlobStorageLimits limits) {
+  if (!limits.IsValid() || manual_limits_set_)
+    return;
+  limits_ = limits;
+}
+
+namespace {
+// Used in UMA metrics, do not change values.
+enum DiskSpaceAdjustmentType {
+  FREEZE_HIT_MIN_AVAILABLE = 0,
+  LOWERED_NEAR_MIN_AVAILABLE = 1,
+  RAISED_NEAR_MIN_AVAILABLE = 2,
+  RESTORED = 3,
+  MAX_ADJUSTMENT_TYPE
+};
+
+enum DiskSpaceAdjustmentStatus { FROZEN, ADJUSTED, NORMAL };
+}  // namespace
+
+void BlobMemoryController::AdjustDiskUsage(uint64_t avail_disk) {
+  DCHECK_LE(disk_used_, limits_.desired_max_disk_space +
+                            limits_.min_available_external_disk_space());
+
+  DiskSpaceAdjustmentStatus curr_status;
+  if (limits_.effective_max_disk_space == limits_.desired_max_disk_space) {
+    curr_status = NORMAL;
+  } else if (limits_.effective_max_disk_space == disk_used_) {
+    curr_status = FROZEN;
+  } else {
+    curr_status = ADJUSTED;
+  }
+  uint64_t old_effective_max_disk_space = limits_.effective_max_disk_space;
+  uint64_t avail_disk_without_blobs = avail_disk + disk_used_;
+
+  if (avail_disk <= limits_.min_available_external_disk_space()) {
+    limits_.effective_max_disk_space = disk_used_;
+    if (curr_status != FROZEN &&

[Mark P, 2017/01/11 17:49:14]

optional nit: parens around binary operators in this test and the ones below

[dmurph, 2017/01/11 18:25:29]

Acknowledged. I prefer w/o parens.

+        limits_.effective_max_disk_space != old_effective_max_disk_space) {
+      UMA_HISTOGRAM_ENUMERATION("Storage.Blob.MaxDiskSpaceAdjustment",
+                                FREEZE_HIT_MIN_AVAILABLE, MAX_ADJUSTMENT_TYPE);
+    }
+  } else if (avail_disk_without_blobs <
+             limits_.min_available_external_disk_space() +
+                 limits_.desired_max_disk_space) {
+    // Subtraction is avoided the if statement to prevent overflows.

[Mark P, 2017/01/11 17:49:14]

I cannot parse this sentence.
Also, it's usually *addition* that causes overflows.

[dmurph, 2017/01/11 18:25:29]

Whoops, meant underflow. Since I moved the addition into the above variable, I
removed the comment.

+    // |effective_max_disk_space| is guaranteed to be less than
+    // |desired_max_disk_space| by the if statement.
+    limits_.effective_max_disk_space =
+        avail_disk_without_blobs - limits_.min_available_external_disk_space();
+    if (curr_status != ADJUSTED &&
+        limits_.effective_max_disk_space != old_effective_max_disk_space) {
+      UMA_HISTOGRAM_ENUMERATION("Storage.Blob.MaxDiskSpaceAdjustment",
+                                curr_status == NORMAL
+                                    ? LOWERED_NEAR_MIN_AVAILABLE
+                                    : RAISED_NEAR_MIN_AVAILABLE,
+                                MAX_ADJUSTMENT_TYPE);
+    }
+  } else {
+    limits_.effective_max_disk_space = limits_.desired_max_disk_space;
+    if (curr_status != NORMAL) {
+      UMA_HISTOGRAM_ENUMERATION("Storage.Blob.MaxDiskSpaceAdjustment", RESTORED,
+                                MAX_ADJUSTMENT_TYPE);
+    }
+  }
+}
+
 base::WeakPtr<QuotaAllocationTask> BlobMemoryController::AppendMemoryTask(
     uint64_t total_bytes_needed,
     std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
     const MemoryQuotaRequestCallback& done_callback) {
   DCHECK(file_paging_enabled_)
       << "Caller tried to reserve memory when CanReserveQuota("
       << total_bytes_needed << ") would have returned false.";
 
   pending_memory_quota_total_size_ += total_bytes_needed;
   pending_memory_quota_tasks_.push_back(
@@ skipping 38 matching lines @@
   return total_items_size.ValueOrDie();
 }
 
 void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
   // Don't do eviction when others are happening, as we don't change our
   // pending_memory_quota_total_size_ value until after the paging files have
   // been written.
   if (pending_evictions_ != 0 || !file_paging_enabled_)
     return;
 
+  uint64_t total_memory_usage =
+      static_cast<uint64_t>(pending_memory_quota_total_size_) +
+      blob_memory_used_;
+
   // We try to page items to disk until our current system size + requested
   // memory is below our size limit.
-  while (pending_memory_quota_total_size_ + blob_memory_used_ >
-         limits_.memory_limit_before_paging()) {
+  // Size limit is a lower |memory_limit_before_paging()| if we have disk space.
+  while (total_memory_usage > limits_.effective_max_disk_space ||
+         (disk_used_ < limits_.effective_max_disk_space &&
+          total_memory_usage > limits_.memory_limit_before_paging())) {
     // We only page when we have enough items to fill a whole page file.
     if (populated_memory_items_bytes_ < limits_.min_page_file_size)
       break;
     DCHECK_LE(limits_.min_page_file_size,
               static_cast<uint64_t>(blob_memory_used_));
 
     std::vector<scoped_refptr<ShareableBlobDataItem>> items_to_swap;
     size_t total_items_size = CollectItemsForEviction(&items_to_swap);
     if (total_items_size == 0)
       break;
@@ skipping 18 matching lines @@
             file_runner_.get());
     // Add the release callback so we decrement our disk usage on file deletion.
     file_reference->AddFinalReleaseCallback(
         base::Bind(&BlobMemoryController::OnBlobFileDelete,
                    weak_factory_.GetWeakPtr(), total_items_size));
 
     // Post the file writing task.
     base::PostTaskAndReplyWithResult(
         file_runner_.get(), FROM_HERE,
         base::Bind(&CreateFileAndWriteItems, blob_storage_dir_,
-                   base::Passed(&page_file_path), file_runner_,
-                   base::Passed(&items_for_paging), total_items_size),
+                   disk_space_function_, base::Passed(&page_file_path),
+                   file_runner_, base::Passed(&items_for_paging),
+                   total_items_size),
         base::Bind(&BlobMemoryController::OnEvictionComplete,
                    weak_factory_.GetWeakPtr(), base::Passed(&file_reference),
                    base::Passed(&items_to_swap), total_items_size));
   }
   RecordTracingCounters();
 }
 
 void BlobMemoryController::OnEvictionComplete(
     scoped_refptr<ShareableFileReference> file_reference,
     std::vector<scoped_refptr<ShareableBlobDataItem>> items,
     size_t total_items_size,
-    FileCreationInfo result) {
+    std::pair<FileCreationInfo, int64_t /* avail_disk */> result) {
   if (!file_paging_enabled_)
     return;
 
-  if (result.error != File::FILE_OK) {
-    DisableFilePaging(result.error);
+  FileCreationInfo& file_info = std::get<0>(result);
+  int64_t avail_disk_space = std::get<1>(result);
+
+  if (file_info.error != File::FILE_OK) {
+    DisableFilePaging(file_info.error);
     return;
   }
 
+  if (avail_disk_space != kUnknownDiskAvailability) {
+    AdjustDiskUsage(static_cast<uint64_t>(avail_disk_space));
+  }
+
   DCHECK_LT(0, pending_evictions_);
   pending_evictions_--;
 
   // Switch item from memory to the new file.
   uint64_t offset = 0;
   for (const scoped_refptr<ShareableBlobDataItem>& shareable_item : items) {
     scoped_refptr<BlobDataItem> new_item(
         new BlobDataItem(base::WrapUnique(new DataElement()), file_reference));
     new_item->data_element_ptr()->SetToFilePathRange(
         file_reference->path(), offset, shareable_item->item()->length(),
-        result.last_modified);
+        file_info.last_modified);
     DCHECK(shareable_item->memory_allocation_);
     shareable_item->set_memory_allocation(nullptr);
     shareable_item->set_item(new_item);
     items_paging_to_file_.erase(shareable_item->item_id());
     offset += shareable_item->item()->length();
   }
   in_flight_memory_used_ -= total_items_size;
 
   // We want callback on blobs up to the amount we've freed.
   MaybeGrantPendingMemoryRequests();
@@ skipping 27 matching lines @@
 uint64_t BlobMemoryController::GetAvailableFileSpaceForBlobs() const {
   if (!file_paging_enabled_)
     return 0;
   // Sometimes we're only paging part of what we need for the new blob, so add
   // the rest of the size we need into our disk usage if this is the case.
   uint64_t total_disk_used = disk_used_;
   if (in_flight_memory_used_ < pending_memory_quota_total_size_) {
     total_disk_used +=
         pending_memory_quota_total_size_ - in_flight_memory_used_;
   }
-  if (limits_.max_blob_disk_space < total_disk_used)
+  if (limits_.effective_max_disk_space < total_disk_used)
     return 0;
-  return limits_.max_blob_disk_space - total_disk_used;
+  return limits_.effective_max_disk_space - total_disk_used;
 }
 
 void BlobMemoryController::GrantMemoryAllocations(
     std::vector<scoped_refptr<ShareableBlobDataItem>>* items,
     size_t total_bytes) {
   // These metrics let us calculate the global distribution of blob storage by
   // subtracting the histograms.
   UMA_HISTOGRAM_COUNTS("Storage.Blob.StorageSizeBeforeAppend",
                        blob_memory_used_ / 1024);
   blob_memory_used_ += total_bytes;
@@ skipping 29 matching lines @@
   MaybeGrantPendingMemoryRequests();
 }
 
 void BlobMemoryController::OnBlobFileDelete(uint64_t size,
                                             const FilePath& path) {
   DCHECK_LE(size, disk_used_);
   disk_used_ -= size;
 }
 
 }  // namespace storage