[BlobAsync] Disk support for blob storage

storage/browser/blob/blob_memory_controller.h

+// Copyright 2016 The Chromium Authors. All rihts reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+#define STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <list>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/containers/mru_cache.h"
+#include "base/files/file.h"
+#include "base/files/file_path.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/time/time.h"
+#include "storage/browser/storage_browser_export.h"
+#include "storage/common/blob_storage/blob_storage_constants.h"
+#include "storage/common/data_element.h"
+
+namespace base {
+class SingleThreadTaskRunner;
+}
+
+namespace storage {
+class ShareableBlobDataItem;
+class ShareableFileReference;
+
+// There should be no memory sizes of 0.
+// This class is responsible for
+// * our current memory usage for blobs, both on disk and in memory,
+// * when we can add a blob w/ a given size,
+// * creating temporary files,
+// * paging blobs to disk.
+class STORAGE_EXPORT BlobMemoryController
+    : public base::SupportsWeakPtr<BlobMemoryController> {

[Marijn Kruisselbrink, 2016/07/08 23:37:37]

In general using a member base::WeakPtrFactory is preferred over deriving from
base::SupportsWeakPtr. SupportsWeakPtr has confusing behavior around destruction
(if any members of the derived class would hold weak ptrs for example). Also
nothing else seems to need weak pointers to BlobMemoryController so keeping weak
pointers an implementation detail seems nicer anyway.

[dmurph, 2016/07/11 23:33:28]

Done.

+ public:
+  enum class MemoryStrategyResult {
+    // We don't have enough memory for this blob.
+    TOO_LARGE,
+    // There isn't any memory that needs transporting.
+    NONE_NEEDED,
+    // Transportation strategies.
+    SHORTCUT,
+    IPC,
+    SHARED_MEMORY,
+    FILE
+  };
+  typedef std::list<std::pair<size_t, base::Callback<void(bool)>>>
+      PendingBlobConstructionList;
+  typedef PendingBlobConstructionList::iterator PendingContructionEntry;

[Marijn Kruisselbrink, 2016/07/08 23:37:37]

Contruction -> Construction

[dmurph, 2016/07/11 23:33:28]

Done.

+
+  struct FileCreationInfo {
+    FileCreationInfo();
+    ~FileCreationInfo();
+    FileCreationInfo(FileCreationInfo&& other);
+    FileCreationInfo& operator=(FileCreationInfo&&);
+
+    base::File::Error error = base::File::FILE_ERROR_FAILED;
+    scoped_refptr<ShareableFileReference> file_reference;
+    base::Optional<base::File> file = base::nullopt;
+    base::Time last_modified;
+  };
+
+  BlobMemoryController();
+  virtual ~BlobMemoryController();
+
+  void EnableDisk(const base::FilePath storage_directory,
+                  scoped_refptr<base::SingleThreadTaskRunner> file_runner);
+
+  // Returns if the data in the descriptions is good data (not a bad IPC). If
+  // SHORTCUT is returned, then the blob can be immediately constructed, and
+  // the a following call to MaybeFitInMemoryNow will always return true.
+  bool DecideBlobTransportationMemoryStrategy(
+      const std::vector<DataElement>& descriptions,
+      uint64_t* total_bytes,
+      MemoryStrategyResult* result) const;
+
+  // Creates a temporary file that we store in our blob temporary directory. The
+  // lifetime of the file is tied to the file_reference.
+  void CreateTemporaryFileForRenderer(
+      uint64_t size_bytes,
+      const base::Callback<void(FileCreationInfo)>& done);
+
+  void FreeMemory(size_t memory_size_bytes);
+
+  enum class MemoryRequestResponse {
+    SPACE_AVAILABLE,
+    SPACE_UNAVAILABLE,
+    PENDING_DISK
+  };
+
+  bool CanFitInSystem(uint64_t size) const;
+
+  // We assume the user called CanFitInSystem before this and that the given
+  // memory size can fit in our system.
+  // Notifies the caller when a blob of the given size can be stored in our blob
+  // system. If the returned entry is not present, that means the blob can fit
+  // right away and the callback won't be called.
+  // The argument on can_request is true if we can create the blob, and
+  // false if there was an error and the new blob needs to be broken.
+  base::Optional<PendingContructionEntry> NotifyWhenMemoryCanPopulated(
+      size_t memory_size_bytes,
+      const base::Callback<void(bool)>& can_request);
+
+  void RemovePendingConstructionEntry(const PendingContructionEntry& entry);
+
+  // This updates the given item in our LRU lookup table.
+  void UpdateBlobItemInRecents(ShareableBlobDataItem* item);
+
+  // This removes the given item from our LRU lookup table.
+  void RemoveBlobItemInRecents(uint64_t id);
+
+  // Disables the disk.
+  void DisableDisk();
+
+  bool disk_enabled() const { return disk_used_; }
+
+  size_t max_ipc_memory_size() const { return max_ipc_memory_size_; }
+  size_t max_shared_memory_size() const { return max_shared_memory_size_; }
+  size_t max_blob_in_memory_size() const { return max_blob_in_memory_size_; }
+  size_t max_blob_memory_space() const { return max_blob_memory_space_; }
+  uint64_t max_blob_disk_space() const { return max_blob_disk_space_; }
+  size_t in_flight_space() const { return in_flight_space_; }
+  uint64_t min_page_file_size() const { return min_page_file_size_; }
+  uint64_t max_file_size() const { return max_file_size_; }
+
+  void SetMemoryContantsForTesting(size_t max_ipc_memory_size,

[Marijn Kruisselbrink, 2016/07/08 23:37:37]

Contants -> Constants

[dmurph, 2016/07/11 23:33:28]

Done.

+                                   size_t max_shared_memory_size,
+                                   size_t max_blob_in_memory_size,
+                                   size_t max_blob_memory_space,
+                                   uint64_t max_blob_disk_space,
+                                   size_t in_flight_space,
+                                   uint64_t min_page_file_size,
+                                   uint64_t max_file_size) {
+    max_ipc_memory_size_ = max_ipc_memory_size;
+    max_shared_memory_size_ = max_shared_memory_size;
+    max_blob_in_memory_size_ = max_blob_in_memory_size;
+    max_blob_memory_space_ = max_blob_memory_space;
+    max_blob_disk_space_ = max_blob_disk_space;
+    in_flight_space_ = in_flight_space;
+    min_page_file_size_ = min_page_file_size;
+    max_file_size_ = max_file_size;
+  }
+
+ private:
+  using RecentItemsCache = base::MRUCache<uint64_t, ShareableBlobDataItem*>;
+
+  // We add a delete callback to the file reference to decrement our disk usage.
+  void OnCreateFile(uint64_t file_size,
+                    const base::Callback<void(FileCreationInfo)>& done,
+                    FileCreationInfo result);
+
+  void MaybeScheduleWaitingBlobs();
+
+  // Returns the size we'll be freeing. We appropriately increment/decrement
+  // the disk, memory, and in flight counts.
+  size_t ScheduleBlobPaging();
+  void SchedulePagingUntilWeCanFit(size_t total_memory_needed);
+
+  // Called when we've completed paging a list of items
+  void OnPagingComplete(
+      std::unique_ptr<std::vector<scoped_refptr<ShareableBlobDataItem>>> items,
+      size_t total_items_size,
+      FileCreationInfo result);
+
+  void RecordTracingCounters();
+
+  size_t GetAvailableMemoryForBlobs() const;
+  uint64_t GetAvailableDiskSpaceForBlobs() const;
+
+  // We decrement the disk space used.
+  void OnBlobFileDelete(uint64_t size, const base::FilePath& path);
+
+  // Memory bookkeeping.
+  size_t blob_memory_used_ = 0;
+  size_t in_flight_memory_used_ = 0;
+  uint64_t disk_used_ = 0;
+
+  scoped_refptr<base::SingleThreadTaskRunner> file_runner_;
+
+  PendingBlobConstructionList blobs_waiting_for_paging_;
+  size_t blobs_waiting_for_paging_size_ = 0;
+
+  // Constants.
+  size_t max_ipc_memory_size_ = kBlobStorageIPCThresholdBytes;
+  size_t max_shared_memory_size_ = kBlobStorageMaxSharedMemoryBytes;
+  size_t max_blob_in_memory_size_ = kBlobStorageMaxBlobMemorySize;
+  size_t max_blob_memory_space_ = kBlobStorageMaxMemoryUsage;
+  uint64_t max_blob_disk_space_ = kBlobStorageMaxDiskSpace;
+  size_t in_flight_space_ = kBlobStorageInFlightMemory;
+  uint64_t min_page_file_size_ = kBlobStorageMinFileSizeBytes;
+  uint64_t max_file_size_ = kBlobStorageMaxFileSizeBytes;
+
+  bool enable_disk_ = false;
+  base::FilePath blob_storage_dir_;
+  uint64_t current_file_num_ = 0;
+
+  // Lifetime of the ShareableBlobDataItem objects is handles externally in the

[Marijn Kruisselbrink, 2016/07/08 23:37:37]

handles -> handled

[dmurph, 2016/07/11 23:33:28]

Done.

+  // BlobStorageContext class.
+  RecentItemsCache recent_item_cache_;
+
+  DISALLOW_COPY_AND_ASSIGN(BlobMemoryController);
+};
+}  // namespace storage
+#endif  // STORAGE_BROWSER_BLOB_BLOB_MEMORY_CONTROLLER_H_