[BlobStorage] BlobMemoryController & tests

storage/browser/blob/blob_memory_controller.h

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+#define STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+
+#include <stdint.h>
+
+#include <list>
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/containers/mru_cache.h"
+#include "base/files/file.h"
+#include "base/files/file_path.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/time/time.h"
+#include "storage/browser/storage_browser_export.h"
+#include "storage/common/blob_storage/blob_storage_constants.h"
+
+namespace base {
+class TaskRunner;
+}
+
+namespace storage {
+class DataElement;
+class ShareableBlobDataItem;
+class ShareableFileReference;
+
+// This class is responsible for file & memory quota bookkeeping, creating files
+// and paging old blob items to disk, and keeping track of an LRU of blob items.
+//
+// See ReserveMemoryQuotaForItems and ReserveFileQuotaForItems for reserving
+// memory and file quota respectively, and use MaybeFreeQuotaForItems to free
+// quota.
+//
+// Use UpdateBlobItemInRecents & RemoveBlobItemInRecents for modifying the LRU
+// of blob items.
+class STORAGE_EXPORT BlobMemoryController {
+ public:
+  enum class Strategy {
+    // We don't have enough memory for this blob.
+    TOO_LARGE,
+    // There isn't any memory that needs transporting.
+    NONE_NEEDED,
+    // Transportation strategies.
+    IPC,
+    SHARED_MEMORY,
+    FILE
+  };
+
+  struct FileCreationInfo {
+    FileCreationInfo();
+    ~FileCreationInfo();
+    FileCreationInfo(FileCreationInfo&& other);
+    FileCreationInfo& operator=(FileCreationInfo&&);
+
+    base::File::Error error = base::File::FILE_ERROR_FAILED;
+    scoped_refptr<ShareableFileReference> file_reference;
+    base::File file;
+    base::Time last_modified;
+  };
+
+  // The bool argument is if we were able to successfuly receive quota.
+  using FileQuotaRequestCallback =
+      base::Callback<void(bool, std::vector<FileCreationInfo>)>;
+  using PendingFileQuotaRequest = uint64_t;
+  static const uint64_t kInvalidFileQuotaRequest = 0;
+
+  // The bool argument is if we were able to successfuly receive quota.
+  using MemoryQuotaRequestCallback = base::Callback<void(bool)>;
+  using PendingBlobConstructionList =
+      std::list<std::pair<size_t, MemoryQuotaRequestCallback>>;
+  using PendingMemoryQuotaRequest = PendingBlobConstructionList::iterator;
+  PendingMemoryQuotaRequest GetInvalidMemoryQuotaRequest();
+
+  BlobMemoryController();
+  virtual ~BlobMemoryController();
+
+  void EnableDisk(const base::FilePath& storage_directory,

[pwnall, 2016/09/16 22:45:24]

Using both disk and file(s) means the reader has to mentally resolve one name to
the other, which is a (minor) drain on processing power. How about using names
like EnableFiles() / DisableFiles() / files_enabled(), and referring to the
concept as "file storage" in the comments?

[dmurph, 2016/09/20 00:00:50]

Good idea, done.

+                  scoped_refptr<base::TaskRunner> file_runner);
+
+  // Disables the disk. This cancels all pending file creations and paging
+  // operations.
+  void DisableDisk();
+
+  bool disk_enabled() const { return disk_enabled_; }
+
+  // Returns the strategy the transportation layer should use to transport the
+  // given memory. |shortcut_bytes| are the number of transport bytes that are
+  // already populated for us, so we don't haved to request them from the
+  // renderer.
+  Strategy DetermineStrategy(size_t shortcut_bytes,
+                             uint64_t total_transportation_bytes) const;
+
+  // Checks to see if we can reserve quota (disk or memory) for the given size.
+  bool CanReserveQuota(uint64_t size) const;
+
+  // This reserves quota for the given |unreserved_memory_items|. We expect

[pwnall, 2016/09/16 22:45:24]

"We expect" seems a bit soft here. Given the DCHECK in the method, I'd expect
something along the lines of "The items must be in the QUOTA_NEEDED state." The
rest of the text could sound like "They will be moved to the QUOTA_REQUESTED
state immediately, and will eventually be placed into the QUOTA_GRANTED state
when |success_callback| is called."

[dmurph, 2016/09/20 00:00:50]

Done.

+  // the item states to start QUOTA_NEEDED, which we change to QUOTA_REQUESTED.
+  // After we reserve memory quota we change their state to QUOTA_GRANTED and
+  // call |success_callback|.  This can happen synchronously.
+  // NOTE: We don't inspect quota limits and assume the user checked
+  //       CanReserveQuota before calling this.
+  // Returns a value if we're async for use with CancelMemoryQuotaReservation.
+  PendingMemoryQuotaRequest ReserveMemoryQuota(
+      std::vector<ShareableBlobDataItem*> unreserved_memory_items,
+      const MemoryQuotaRequestCallback& success_callback);
+
+  void CancelMemoryQuotaReservation(const PendingMemoryQuotaRequest& entry);
+
+  // This reserves quota for the given |unreserved_file_items|. We expect
+  // the item states to start QUOTA_NEEDED, which we change to QUOTA_REQUESTED.
+  // After we reserve memory quota we change their state to QUOTA_GRANTED and
+  // call |success_callback|.  All items should be temporary file items
+  // (BlobDataBuilder::IsTemporaryFileItem returns true).
+  // NOTE: We don't inspect quota limits and assume the user checked
+  //       CanReserveQuota before calling this.
+  // The return value can be used with CancelFileQuotaReservation to cancel.
+  PendingFileQuotaRequest ReserveFileQuota(
+      std::vector<ShareableBlobDataItem*> unreserved_file_items,
+      const FileQuotaRequestCallback& success_callback);
+
+  void CancelFileQuotaReservation(const PendingFileQuotaRequest& entry);
+
+  // This frees quota for items that don't have any blob references.
+  void MaybeFreeQuotaForItems(
+      const std::vector<scoped_refptr<ShareableBlobDataItem>>& items);
+
+  // This is used to release quota for an item that was supposed to be filled
+  // with a data copy from an item in another blob, but by the time the copy
+  // was meant to be performed the original item had been paged to file. This
+  // functionally just decrements the memory by the item length, but we do
+  // extra checks to make sure our state is correct.
+  void FreeQuotaForPagedItemReference(
+      const scoped_refptr<ShareableBlobDataItem>& item);
+
+  // This adds or updates a bytes item in our LRU table used for paging items
+  // to disk. The item must have state POPULATED_WITH_QUOTA.
+  void UpdateBlobItemInRecents(ShareableBlobDataItem* item);
+  void RemoveBlobItemInRecents(const ShareableBlobDataItem& item);
+
+  size_t memory_usage() const {
+    return blob_memory_used_ + in_flight_memory_used_;
+  }
+  uint64_t disk_usage() const { return disk_used_; }
+
+  const BlobStorageLimits& limits() const { return limits_; }
+  void SetLimitsForTesting(BlobStorageLimits limits) { limits_ = limits; }
+
+ private:
+  // Must be class method because state_ is private in ShareableBlobDataItem.
+  void SetStateQuotaGrantedAndCallback(
+      std::vector<scoped_refptr<ShareableBlobDataItem>> items,
+      const BlobMemoryController::MemoryQuotaRequestCallback& final_callback,
+      bool success);
+
+  // Called when we've finished creating files for ReserveQuotaForItems when the
+  // items are pending file items. We handle error cases, the case where the

[pwnall, 2016/09/16 22:45:24]

the caller?

[dmurph, 2016/09/20 00:00:50]

Fixed this documentation.

+  // cancels the quota reservation, and the success case where we add a delete
+  // callback to the file reference to decrement our disk usage before fowarding
+  // the files and result to the |file_callback|.
+  void OnCreateFiles(
+      std::vector<uint64_t> file_sizes,
+      std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items,
+      uint64_t disk_quota_entry,
+      const FileQuotaRequestCallback& file_callback,
+      std::vector<FileCreationInfo> result);
+
+  void MaybeGrantPendingQuotaRequests();
+
+  // We schedule paging until our memory usage is below our memory limit. We use
+  // the lru table to find old items, and we combine them until we reach the
+  // min_page_file_size(), and don't schedule paging until we have at least that
+  // amount of memory to save to disk.
+  void MaybeSchedulePagingUntilSystemHealthy();
+
+  // Called when we've completed paging a list of items to disk. This is where
+  // we swap the bytes items for file items, and and update our bookkeeping.
+  void OnPagingComplete(
+      std::unique_ptr<std::vector<scoped_refptr<ShareableBlobDataItem>>> items,
+      size_t total_items_size,
+      FileCreationInfo result);
+
+  void RecordTracingCounters();
+
+  size_t GetAvailableMemoryForBlobs() const;
+  uint64_t GetAvailableDiskSpaceForBlobs() const;
+
+  // This is registered as a callback for file deletions on the file reference
+  // of our paging files. We decrement the disk space used.
+  void OnBlobFileDelete(uint64_t size, const base::FilePath& path);
+
+  BlobStorageLimits limits_;
+
+  // Memory bookkeeping. These numbers are all disjoint.
+  // This is the amount of memory we're using for blobs in RAM.
+  size_t blob_memory_used_ = 0;
+  // This is memory we're temporarily using while we try to write blob items to
+  // disk.
+  size_t in_flight_memory_used_ = 0;
+  // This is the amount of memory we're using on disk.
+  uint64_t disk_used_ = 0;
+
+  size_t pending_pagings_ = 0;
+  PendingBlobConstructionList blobs_waiting_for_paging_;
+  size_t blobs_waiting_for_paging_size_ = 0;
+
+  // We use the same data storage as above to keep the PendingEntry API
+  // consistent.
+  uint64_t curr_disk_save_entry_ = 0;
+  std::unordered_map<uint64_t, uint64_t> pending_file_request_sizes_;
+
+  scoped_refptr<base::TaskRunner> file_runner_;
+
+  bool disk_enabled_ = false;
+  base::FilePath blob_storage_dir_;
+  uint64_t current_file_num_ = 0;
+
+  // Lifetime of the ShareableBlobDataItem objects is handled externally in the
+  // BlobStorageContext class.
+  base::MRUCache<uint64_t, ShareableBlobDataItem*> recent_item_cache_;
+  size_t recent_item_cache_bytes_ = 0;
+  // We need to keep track of items currently being paged to disk so that if
+  // another blob successfully grabs a ref, we can prevent it from adding the
+  // item to the recent_item_cache_ above.
+  std::unordered_set<uint64_t> items_saving_to_disk_;
+
+  base::WeakPtrFactory<BlobMemoryController> ptr_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(BlobMemoryController);
+};
+}  // namespace storage
+#endif  // STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_