[BlobAsync] Disk support for blob storage

storage/browser/blob/blob_memory_controller.h

Index: storage/browser/blob/blob_memory_controller.h
diff --git a/storage/browser/blob/blob_memory_controller.h b/storage/browser/blob/blob_memory_controller.h
new file mode 100644
index 0000000000000000000000000000000000000000..e2e95763061132bc7d4c9513e9628c608b9ba8c7
--- /dev/null
+++ b/storage/browser/blob/blob_memory_controller.h
@@ -0,0 +1,307 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+#define STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <list>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/containers/hash_tables.h"
+#include "base/containers/mru_cache.h"
+#include "base/files/file.h"
+#include "base/files/file_path.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/time/time.h"
+#include "storage/browser/storage_browser_export.h"
+#include "storage/common/blob_storage/blob_storage_constants.h"
+
+namespace base {
+class TaskRunner;
+}
+
+namespace storage {
+class DataElement;
+class ShareableBlobDataItem;
+class ShareableFileReference;
+
+// This class is responsible for all memory, quota, and file operations for the
+// blob system. To determine the transportation strategy for a blob transfer
+// from the renderer, use DecideBlobTransportationMemoryStrategy. To request
+// quota and create files for blob items, you can use:
+// * CanFitInSystem to see if we can fit at all,
+// * ReserveQuotaForItems to ask for quota reservation (for either bytes or
+//   files, but not in the same call), and
+// * CancelQuotaReservation to optionally cancel the quota request before it is
+//   granted.
+// When destroying a blob, one can call MaybeFreeQuotaForItems to free unused
+// bytes items. Finally, one can use UpdateBlobItemInRecents and
+// RemoveBlobItemInRecents to add/update and remove populated bytes items as
+// candiates for paging memory to disk. This class automatically manages paging
+// memory to disk if we are running low on quota.

[michaeln, 2016/08/15 22:44:44]

Please be less verbose in this comment (and in general where possible). I'd
suggest dropping the method specific verbiage.

[dmurph, 2016/08/19 00:18:33]

Done.

+class STORAGE_EXPORT BlobMemoryController {
+ public:
+  enum class MemoryStrategyResult {

[michaeln, 2016/08/15 22:44:44]

Shorter symbols, maybe MemoryStrategy or even just Strategy. Since the name is
scoped to BlobMemoryController, Strategy is probably sufficient.

[dmurph, 2016/08/19 00:18:33]

Done.

+    // We don't have enough memory for this blob.
+    TOO_LARGE,
+    // There isn't any memory that needs transporting.
+    NONE_NEEDED,
+    // Transportation strategies.
+    IPC,
+    SHARED_MEMORY,
+    FILE
+  };
+
+  struct FileCreationInfo {
+    FileCreationInfo();
+    ~FileCreationInfo();
+    FileCreationInfo(FileCreationInfo&& other);
+    FileCreationInfo& operator=(FileCreationInfo&&);
+
+    base::File::Error error = base::File::FILE_ERROR_FAILED;
+    scoped_refptr<ShareableFileReference> file_reference;
+    base::File file;
+    base::Time last_modified;
+  };
+
+  using QuotaRequestCallback =
+      base::Callback<void(bool, std::vector<FileCreationInfo>)>;
+  using PendingBlobConstructionList =
+      std::list<std::pair<size_t, QuotaRequestCallback>>;
+
+  static const size_t kInvalidDiskQuotaEntry = 0;
+
+  // This struct keeps track of a quota reservation, which the user can use to
+  // cancel the quota request with CancelQuotaReservation. This becomes invalid
+  // after |success_callback| is called from the ReserveQuotaForItems call.

[michaeln, 2016/08/15 22:44:44]

Does this need to be public? would be nice if the details of this were opaque to
the caller to simplify the interface.

[dmurph, 2016/08/19 00:18:33]

Simplified

+  struct PendingQuotaEntry {
+    explicit PendingQuotaEntry(uint64_t disk_quota_entry);
+    explicit PendingQuotaEntry(
+        PendingBlobConstructionList::iterator memory_quota_entry);
+    PendingQuotaEntry(const PendingQuotaEntry& other);
+    ~PendingQuotaEntry();
+
+    // We're either saving

[michaeln, 2016/08/15 22:44:44]

i don't understand the comment?

[dmurph, 2016/08/19 00:18:33]

Removed.

+    uint64_t disk_quota_entry = kInvalidDiskQuotaEntry;
+    PendingBlobConstructionList::iterator memory_quota_entry;
+  };
+
+  BlobMemoryController();
+  virtual ~BlobMemoryController();
+
+  void EnableDisk(const base::FilePath& storage_directory,
+                  scoped_refptr<base::TaskRunner> file_runner);
+
+  // Disables the disk. This cancels all pending file creations and paging
+  // operations.
+  void DisableDisk();
+
+  bool disk_enabled() const { return disk_enabled_; }
+
+  // Returns the strategy the transportation layer should use to transport the
+  // given memory. |shortcut_bytes| are the number of transport bytes that are
+  // already populated for us, so we don't haved to request them from the
+  // renderer.
+  MemoryStrategyResult DecideBlobTransportationMemoryStrategy(

[michaeln, 2016/08/15 22:44:44]

Please use shorter symbols: DetermineStrategy() DecideStrategy() PickStrategy()
or ChooseStrategy().

[dmurph, 2016/08/19 00:18:33]

Done.

+      size_t shortcut_bytes,
+      uint64_t total_transportation_bytes) const;
+
+  // Checks to see if the given size can fit in the system.
+  bool CanFitInSystem(uint64_t size) const;

[michaeln, 2016/08/15 22:44:44]

Does this need to be public? It's checked when the strategy is chosen and again
when the pending blob is created. Why does it need to be public?

[dmurph, 2016/08/19 00:18:33]

When we create a blob by using BlobStorageContext::AddFinishedBlob, we don't
check a transportation strategy, and so we still need to check this.

+
+  // 'Reserves' space for the given items, and calls |success_callback| when

[michaeln, 2016/08/15 22:44:44]

why the quotes?

[dmurph, 2016/08/19 00:18:33]

Simplified.

+  // we're done. This can be done synchronously before the method returns.
+  // We only reserve for items ShareableBlobDataItem::state() == QUOTA_NEEDED.
+  // We change this to QUOTA_REQUESTED while we allocate space, and
+  // QUOTA_GRANTED when we've succeeded and before we call |success_callback|.
+  // If the return value is populated, you can use that to cancel the quota
+  // reservation using CancelQuotaReservation.
+  // NOTE: We assume the user checked CanFitInSystem before this, as we don't
+  //       inspect quotas.
+  //
+  // One can only reserve quota for bytes and temporary file items (see

[michaeln, 2016/08/15 22:44:44]

Maybe make two methods them, one to ReserveMemoryQuota and the other to
ResourceDiskQuota.

[dmurph, 2016/08/19 00:18:33]

Great idea!

+  // BlobDataBuilder). You CANNOT ask for quota for both types of items at the
+  // same time, and you must split up your requests. When asking for temporary
+  // file item quota, the files will be created and the FileCreationInfo vector
+  // in the |success_callback| will be populated with the files.
+  base::Optional<PendingQuotaEntry> ReserveQuotaForItems(

[michaeln, 2016/08/15 22:44:44]

Can this return an integer id type to represent the reservation? I think the use
of optional and exposing of the struct with internal details adds complexity to
this already complicated public interface.

[dmurph, 2016/08/19 00:18:33]

Yes, done.

+      std::vector<ShareableBlobDataItem*> items,
+      const QuotaRequestCallback& success_callback);
+
+  // If we no longer need memory requsted from ReserveQuotaForItems before
+  // |success_callback| has been called above, you can call this method to
+  // correctly free the requested memory.
+  void CancelQuotaReservation(const PendingQuotaEntry& entry);
+
+  // This looks through the given items and releases quota for any items that:
+  // * Are bytes items,
+  // * don't have any blobs referencing them, AND
+  // * were allocated quota originally (state is POPULATED_WITH_QUOTA).
+  // We don't worry about file items as we use the ShareableFileReference
+  // destruction callback feature to detect when those are deleted.
+  void MaybeFreeQuotaForItems(
+      const std::vector<scoped_refptr<ShareableBlobDataItem>>& items);
+
+  // This is used to release quota for an item that was supposed to be filled
+  // with a data copy from an item in another blob, but by the time the copy
+  // was meant to be performed the original item had been paged to file. This
+  // functionally just decrements the memory by the item length, but we do
+  // extra checks to make sure our state is correct.
+  void FreeQuotaForPagedItemReference(
+      const scoped_refptr<ShareableBlobDataItem>& item);
+
+  // This adds or updates the given item in our LRU table used for paging items
+  // to disk. The item must be a bytes item that is both populated an has it's
+  // quota allocated (AKA, in state POPULATED_WITH_QUOTA).
+  void UpdateBlobItemInRecents(ShareableBlobDataItem* item);
+
+  // This removes the given item from our LRU table used for paging.
+  void RemoveBlobItemInRecents(const ShareableBlobDataItem& item);
+
+  size_t memory_usage() const {
+    return blob_memory_used_ + in_flight_memory_used_;
+  }
+
+  uint64_t disk_usage() const { return disk_used_; }
+
+  size_t max_ipc_memory_size() const { return max_ipc_memory_size_; }
+  size_t max_shared_memory_size() const { return max_shared_memory_size_; }
+  size_t max_blob_in_memory_size() const { return max_blob_in_memory_size_; }
+  uint64_t max_blob_disk_space() const { return max_blob_disk_space_; }
+  size_t in_flight_space() const { return in_flight_space_; }
+  uint64_t min_page_file_size() const { return min_page_file_size_; }
+  uint64_t max_file_size() const { return max_file_size_; }
+
+  void SetMemoryConstantsForTesting(size_t max_ipc_memory_size,
+                                    size_t max_shared_memory_size,
+                                    size_t max_blob_in_memory_size,
+                                    uint64_t max_blob_disk_space,
+                                    size_t in_flight_space,
+                                    uint64_t min_page_file_size,
+                                    uint64_t max_file_size) {
+    max_ipc_memory_size_ = max_ipc_memory_size;
+    max_shared_memory_size_ = max_shared_memory_size;
+    max_blob_in_memory_size_ = max_blob_in_memory_size;
+    max_blob_disk_space_ = max_blob_disk_space;
+    in_flight_space_ = in_flight_space;
+    min_page_file_size_ = min_page_file_size;
+    max_file_size_ = max_file_size;
+  }
+
+ private:
+  // This is a helper callback method that sets the state of the given |items|
+  // to POPULATED_WITH_QUOTA before then forwarding |success| and |files| to the
+  // |final_callback|.
+  void SetStateQuotaGrantedAndCallback(
+      std::vector<scoped_refptr<ShareableBlobDataItem>> items,
+      const BlobMemoryController::QuotaRequestCallback& final_callback,
+      bool success,
+      std::vector<FileCreationInfo> files);
+
+  // Called when we've finished creating files for ReserveQuotaForItems when the
+  // items are pending file items. We handle error cases, the case where the
+  // cancels the quota reservation, and the success case where we add a delete
+  // callback to the file reference to decrement our disk usage before fowarding
+  // the files and result to the |file_callback|.
+  void OnCreateFiles(
+      std::vector<uint64_t> file_sizes,
+      std::vector<scoped_refptr<ShareableBlobDataItem>> pending_items,
+      uint64_t disk_quota_entry,
+      const QuotaRequestCallback& file_callback,
+      std::vector<FileCreationInfo> result);
+
+  void MaybeGrantPendingQuotaRequests();
+
+  // We schedule paging until our memory usage is below our memory limit. We use
+  // the lru table to find old items, and we combine them until we reach the
+  // min_page_file_size(), and don't schedule paging until we have at least that
+  // amount of memory to save to disk.
+  void MaybeSchedulePagingUntilSystemHealthy();
+
+  // Called when we've completed paging a list of items to disk. This is where
+  // we swap the bytes items for file items, and and update our bookkeeping.
+  void OnPagingComplete(
+      std::unique_ptr<std::vector<scoped_refptr<ShareableBlobDataItem>>> items,
+      size_t total_items_size,
+      FileCreationInfo result);
+
+  void RecordTracingCounters();
+
+  size_t GetAvailableMemoryForBlobs() const;
+  uint64_t GetAvailableDiskSpaceForBlobs() const;
+
+  // This is registered as a callback for file deletions on the file reference
+  // of our paging files. We decrement the disk space used.
+  void OnBlobFileDelete(uint64_t size, const base::FilePath& path);
+
+  // Memory bookkeeping. These numbers are all disjoint.
+  // This is the amount of memory we're using for blobs in RAM.
+  size_t blob_memory_used_ = 0;
+  // This is memory we're temporarily using while we try to write blob items to
+  // disk.
+  size_t in_flight_memory_used_ = 0;
+  // This is the amount of memory we're using on disk.
+  uint64_t disk_used_ = 0;
+
+  size_t pending_pagings_ = 0;
+  PendingBlobConstructionList blobs_waiting_for_paging_;
+  size_t blobs_waiting_for_paging_size_ = 0;
+
+  // We use the same data storage as above to keep the PendingEntry API
+  // consistent.
+  uint64_t curr_disk_save_entry_ = 0;
+  base::hash_set<size_t> pending_file_opens_;

[Marijn Kruisselbrink, 2016/08/05 23:23:55]

base::hash_set is deprecated, use std::unordered_set instead

[dmurph, 2016/08/19 00:18:33]

Done.

+
+  scoped_refptr<base::TaskRunner> file_runner_;
+
+  // Constants.
+  // TODO(dmurph): consolidate these into a struct.

[michaeln, 2016/08/15 22:44:44]

can this be done in this cl?

[dmurph, 2016/08/19 00:18:33]

Done.

+  // This is the maximum amount of memory we can send in an IPC.
+  size_t max_ipc_memory_size_ = kBlobStorageIPCThresholdBytes;
+  // This is the maximum size of a shared memory handle.
+  size_t max_shared_memory_size_ = kBlobStorageMaxSharedMemoryBytes;
+  // This is the memory we can allocated towards blobs stored in RAM.
+  size_t max_blob_in_memory_size_ = kBlobStorageMaxBlobMemorySpace;
+  // This is the maximum amount of disk space we can use.
+  uint64_t max_blob_disk_space_ = kBlobStorageMaxDiskSpace;
+  // This is the maximum amount of memory we can temporarily allocate towards
+  // blob items that we're currently writing to disk.
+  size_t in_flight_space_ = kBlobStorageInFlightSpace;
+  // This is the minimum file size we can use when paging blob items to disk.
+  // We combine items until we reach at least this size.
+  uint64_t min_page_file_size_ = kBlobStorageMinFileSizeBytes;
+  // This is the maximum file size we can create. We use this is
+  // BlobAsyncTransportHost to segment large blobs, and then we create the files
+  // using CreateTemporaryFile.
+  uint64_t max_file_size_ = kBlobStorageMaxFileSizeBytes;
+
+  bool disk_enabled_ = false;
+  base::FilePath blob_storage_dir_;
+  uint64_t current_file_num_ = 0;
+
+  // Lifetime of the ShareableBlobDataItem objects is handled externally in the
+  // BlobStorageContext class.
+  base::MRUCache<uint64_t, ShareableBlobDataItem*> recent_item_cache_;
+  size_t recent_item_cache_bytes_ = 0;
+  // We need to keep track of items currently being paged to disk so that if
+  // another blob successfully grabs a ref, we can prevent it from adding the
+  // item to the recent_item_cache_ above.
+  base::hash_set<uint64_t> items_saving_to_disk_;
+
+  base::WeakPtrFactory<BlobMemoryController> ptr_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(BlobMemoryController);
+};
+}  // namespace storage
+#endif  // STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_