[BlobStorage] BlobMemoryController & tests

storage/browser/blob/blob_memory_controller.h

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+#define STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+
+#include <stdint.h>
+
+#include <list>
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/containers/mru_cache.h"
+#include "base/files/file.h"
+#include "base/files/file_path.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/time/time.h"
+#include "storage/browser/storage_browser_export.h"
+#include "storage/common/blob_storage/blob_storage_constants.h"
+
+namespace base {
+class TaskRunner;
+}
+
+namespace storage {
+class DataElement;
+class ShareableBlobDataItem;
+class ShareableFileReference;
+
+// This class's main responsability is deciding how blob data gets stored.

[kinuko, 2016/10/12 15:26:42]

responsability -> responsibility

[dmurph, 2016/10/13 00:39:31]

Done.

+// This encompasses:
+// * Keeping track of memory & file quota,
+// * How to transport the blob data from the renderer (DetermineStrategy),
+// * Allocating memory & file quota (ReserveMemoryQuota, ReserveFileQuota)
+// * Freeing quota (MaybeFreeQuotaForItems, ForceFreeMemoryQuotaForItem)
+// * Paging memory quota to disk when we're nearing our memory limit, and
+// * Maintaining an LRU of memory items to choose candidates to page to disk
+//   (NotifyMemoryItemUsed, *FreeMemoryQuotaForItem*).
+// This class can only be interacted with on the IO thread.
+class STORAGE_EXPORT BlobMemoryController {
+ public:
+  enum class Strategy {
+    // We don't have enough memory for this blob.
+    TOO_LARGE,
+    // There isn't any memory that needs transporting.
+    NONE_NEEDED,
+    // Transportation strategies.
+    IPC,
+    SHARED_MEMORY,
+    FILE
+  };
+
+  struct FileCreationInfo {
+    FileCreationInfo();
+    ~FileCreationInfo();
+    FileCreationInfo(FileCreationInfo&& other);
+    FileCreationInfo& operator=(FileCreationInfo&&);
+
+    base::File::Error error = base::File::FILE_ERROR_FAILED;
+    base::File file;
+    scoped_refptr<base::TaskRunner> file_deletion_runner;
+    base::FilePath path;
+    scoped_refptr<ShareableFileReference> file_reference;
+    base::Time last_modified;
+  };
+
+  class QuotaAllocationTask {
+   public:
+    // Operation is cancelled and the callback will NOT be called. This object
+    // will be destroyed by this call.
+    virtual void Cancel() = 0;
+
+   protected:
+    virtual ~QuotaAllocationTask();
+  };
+
+  // The bool argument is true if we successfully received memory quota.
+  using MemoryQuotaRequestCallback = base::Callback<void(bool)>;
+  // The bool argument is true if we successfully received file quota, and the
+  // vector argument provides the file info.
+  using FileQuotaRequestCallback =
+      base::Callback<void(bool, std::vector<FileCreationInfo>)>;
+
+  // We enable file paging if |file_runner| isn't a nullptr.
+  BlobMemoryController(const base::FilePath& storage_directory,
+                       scoped_refptr<base::TaskRunner> file_runner);
+  virtual ~BlobMemoryController();
+
+  // Disables the disk. This cancels all pending file creations and paging

[kinuko, 2016/10/12 15:26:42]

nit: disk -> file paging
for consistency?

[dmurph, 2016/10/13 00:39:31]

Done.

+  // operations.
+  void DisableFilePaging();
+
+  bool file_paging_enabled() const { return file_paging_enabled_; }
+
+  // Returns the strategy the transportation layer should use to transport the
+  // given memory. |preemptive_transported_bytes| are the number of transport
+  // bytes that are already populated for us, so we don't haved to request them
+  // from the renderer.
+  Strategy DetermineStrategy(size_t preemptive_transported_bytes,
+                             uint64_t total_transportation_bytes) const;
+
+  // Checks to see if we can reserve quota (disk or memory) for the given size.
+  bool CanReserveQuota(uint64_t size) const;
+
+  // Reserves quota for the given |unreserved_memory_items|. The items must be
+  // bytes items in QUOTA_NEEDED state, which we change to QUOTA_REQUESTED.
+  // After we reserve memory quota we change their state to QUOTA_GRANTED and
+  // call |done_callback|.  This can happen synchronously.
+  // Returns a task handle if the request is asynchronous for cancellation.
+  // NOTE: We don't inspect quota limits and assume the user checked
+  //       CanReserveQuota before calling this.
+  base::WeakPtr<QuotaAllocationTask> ReserveMemoryQuota(
+      std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
+      const MemoryQuotaRequestCallback& done_callback);
+
+  // Reserves quota for the given |unreserved_file_items|. The items must be
+  // temporary file items (BlobDataBuilder::IsTemporaryFileItem returns true) in
+  // QUOTA_NEEDED state, which we change to QUOTA_REQUESTED. After we reserve
+  // file quota we change their state to QUOTA_GRANTED and call
+  // |done_callback|.
+  // Returns a task handle for cancellation.
+  // NOTE: We don't inspect quota limits and assume the user checked
+  //       CanReserveQuota before calling this.
+  base::WeakPtr<QuotaAllocationTask> ReserveFileQuota(
+      std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_file_items,
+      const FileQuotaRequestCallback& done_callback);
+
+  // This frees quota for items that are not referenced by any blobs.
+  void MaybeFreeMemoryQuotaForItems(

[michaeln, 2016/10/11 20:55:24]

What do think about making the "free" call a function of the
ShareableBlobDataItem? That might eliminate the maybeness and be less error
prone. It's nice the file quota is returned automatically when the file is no
longer needed. I think would be nice of mem quota was similarly reclaimed
automatically without the caller having to do something extra.

[dmurph, 2016/10/13 00:39:31]

Done.

+      const std::vector<scoped_refptr<ShareableBlobDataItem>>& items);
+  // Forcibly frees memory quota for given item. Item must have granted quota.
+  void ForceFreeMemoryQuotaForItem(

[michaeln, 2016/10/11 20:55:24]

what is the usecase for this method? i don't see it being used

[dmurph, 2016/10/13 00:39:31]

When we have created (and asked quota for) a 'future' bytes item which we'll use
to copy data from a referenced blob. When we go to do the copy, it could have
been paged to disk, which results in a file item that we no longer need to copy
from. So we need to free that memory and just do a file item reference.


W/ the addition of the allocation callback in the shareabledataitem, this is
unneeded.

+      const scoped_refptr<ShareableBlobDataItem>& item);
+
+  // Called when initially populated or upon later access.
+  void NotifyMemoryItemUsed(ShareableBlobDataItem* item);
+
+  size_t memory_usage() const {
+    return blob_memory_used_ + in_flight_memory_used_;
+  }
+  uint64_t disk_usage() const { return disk_used_; }
+
+  const BlobStorageLimits& limits() const { return limits_; }
+  void set_limits_for_testing(BlobStorageLimits limits) { limits_ = limits; }

[kinuko, 2016/10/12 15:26:42]

nit: const BlobStorageLimits& limits

[dmurph, 2016/10/13 00:39:31]

Done.

+
+ private:
+  template <typename T>
+  class BaseQuotaAllocationTask;
+  class FileQuotaAllocationTask;
+  class MemoryQuotaAllocationTask;
+
+  using PendingMemoryQuotaTaskList =
+      std::list<std::unique_ptr<MemoryQuotaAllocationTask>>;
+  using PendingFileQuotaTaskList =
+      std::list<std::unique_ptr<FileQuotaAllocationTask>>;
+
+  base::WeakPtr<QuotaAllocationTask> AppendMemoryTask(
+      uint64_t total_bytes_needed,
+      std::vector<scoped_refptr<ShareableBlobDataItem>> unreserved_memory_items,
+      const MemoryQuotaRequestCallback& done_callback);
+
+  // Removes the given blob item from our LRU list for paging.
+  void RemoveItemInRecents(const ShareableBlobDataItem& item);
+
+  void MaybeGrantPendingQuotaRequests();
+
+  size_t CollectItemsForEviction(
+      std::vector<scoped_refptr<ShareableBlobDataItem>>* output);
+
+  // Schedule paging until our memory usage is below our memory limit.
+  void MaybeScheduleEvictionUntilSystemHealthy();
+
+  // Called when we've completed evicting a list of items to disk. This is where
+  // we swap the bytes items for file items, and and update our bookkeeping.
+  void OnEvictionComplete(
+      scoped_refptr<ShareableFileReference> file_reference,
+      std::vector<scoped_refptr<ShareableBlobDataItem>> items,
+      size_t total_items_size,
+      FileCreationInfo result);
+
+  size_t GetAvailableMemoryForBlobs() const;
+  uint64_t GetAvailableFileSpaceForBlobs() const;
+
+  // This is registered as a callback for file deletions on the file reference
+  // of our paging files. We decrement the disk space used.
+  void OnBlobFileDelete(uint64_t size, const base::FilePath& path);
+
+  base::FilePath GenerateNextPageFileName();
+
+  // This records diagnostic counters of our memory quotas. Called when usage
+  // changes.
+  void RecordTracingCounters() const;
+
+  BlobStorageLimits limits_;
+
+  // Memory bookkeeping. These numbers are all disjoint.
+  // This is the amount of memory we're using for blobs in RAM.
+  size_t blob_memory_used_ = 0;
+  // This is memory we're temporarily using while we try to write blob items to
+  // disk.
+  size_t in_flight_memory_used_ = 0;
+  // This is the amount of memory we're using on disk.
+  uint64_t disk_used_ = 0;
+
+  // State for GenerateNextPageFileName.
+  uint64_t current_file_num_ = 0;
+
+  size_t pending_memory_quota_total_size_ = 0;
+  PendingMemoryQuotaTaskList pending_memory_quota_tasks_;
+  PendingFileQuotaTaskList pending_file_quota_tasks_;
+
+  size_t pending_evictions_ = 0;
+
+  bool file_paging_enabled_ = false;
+  base::FilePath blob_storage_dir_;
+  scoped_refptr<base::TaskRunner> file_runner_;
+
+  // Lifetime of the ShareableBlobDataItem objects is handled externally in the
+  // BlobStorageContext class.
+  base::MRUCache<uint64_t, ShareableBlobDataItem*> recent_item_cache_;
+  size_t recent_item_cache_bytes_ = 0;
+  // We need to keep track of items currently being paged to disk so that if
+  // another blob successfully grabs a ref, we can prevent it from adding the
+  // item to the recent_item_cache_ above.
+  std::unordered_set<uint64_t> items_saving_to_disk_;

[kinuko, 2016/10/12 15:26:42]

nit: saving_to_disk_ or paging_to_file_ ?

[dmurph, 2016/10/13 00:39:31]

Done.

+
+  // We clear this when disabling paging to disable pending file tasks.
+  base::WeakPtrFactory<BlobMemoryController> file_task_factory_;
+  base::WeakPtrFactory<BlobMemoryController> file_destruction_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(BlobMemoryController);
+};
+}  // namespace storage
+#endif  // STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_