[BlobAsync] Disk support for blob storage

storage/browser/blob/blob_storage_context.h

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_
 #define STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <map>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "base/callback_forward.h"
 #include "base/gtest_prod_util.h"
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "storage/browser/blob/blob_data_handle.h"
+#include "storage/browser/blob/blob_memory_controller.h"
 #include "storage/browser/blob/blob_storage_registry.h"
-#include "storage/browser/blob/internal_blob_data.h"
 #include "storage/browser/storage_browser_export.h"
 #include "storage/common/blob_storage/blob_storage_constants.h"
-#include "storage/common/data_element.h"
 
 class GURL;
 
 namespace base {
 class FilePath;
 class Time;
+class TaskRunner;
 }
 
 namespace content {
 class BlobDispatcherHost;
 class BlobDispatcherHostTest;
 }
 
 namespace storage {
 
 class BlobDataBuilder;
+class BlobDataHandle;
 class BlobDataItem;
 class BlobDataSnapshot;
 class ShareableBlobDataItem;
 
-// This class handles the logistics of blob Storage within the browser process,
-// and maintains a mapping from blob uuid to the data. The class is single
-// threaded and should only be used on the IO thread.
-// In chromium, there is one instance per profile.
+// This class handles the logistics of blob storage within the browser process.
+// We are single threaded and should only be used on the IO thread. In Chromium
+// there is one instance per profile.
 class STORAGE_EXPORT BlobStorageContext
     : public base::SupportsWeakPtr<BlobStorageContext> {
  public:
+  using PopulatationAllowedCallback =
+      base::Callback<void(BlobStatus,
+                          std::vector<BlobMemoryController::FileCreationInfo>)>;
+
   BlobStorageContext();
   ~BlobStorageContext();
 
   std::unique_ptr<BlobDataHandle> GetBlobDataFromUUID(const std::string& uuid);
   std::unique_ptr<BlobDataHandle> GetBlobDataFromPublicURL(const GURL& url);
 
-  // Useful for coining blobs from within the browser process. If the
-  // blob cannot be added due to memory consumption, returns NULL.
-  // A builder should not be used twice to create blobs, as the internal
-  // resources are refcounted instead of copied for memory efficiency.
-  // To cleanly use a builder multiple times, please call Clone() on the
-  // builder, or even better for memory savings, clear the builder and append
-  // the previously constructed blob.
+  // Useful for coining blobs from within the browser process. Use the
+  // BlobDataHandle::GetBlobStatus call to check if there was an error. If the
+  // builder contains items added by AppendFutureData or AppendFutureFile, then
+  // you must use the BuildBlob method below.

[michaeln, 2016/08/15 22:44:44]

Might not need the comments about a finished blob not containing future items.
Less is more. I think the impl could DCHECK that condition.

[dmurph, 2016/08/19 00:18:33]

Done.

   std::unique_ptr<BlobDataHandle> AddFinishedBlob(
       const BlobDataBuilder& builder);
 
-  // Deprecated, use const ref version above.
+  // Deprecated, use const ref version above or BuildBlob below.
   std::unique_ptr<BlobDataHandle> AddFinishedBlob(
       const BlobDataBuilder* builder);
 
+  std::unique_ptr<BlobDataHandle> AddBrokenBlob(
+      const std::string& uuid,
+      const std::string& content_type,
+      const std::string& content_disposition,
+      BlobStatus reason);
+
   // Useful for coining blob urls from within the browser process.
   bool RegisterPublicBlobURL(const GURL& url, const std::string& uuid);
   void RevokePublicBlobURL(const GURL& url);
 
-  size_t memory_usage() const { return memory_usage_; }
   size_t blob_count() const { return registry_.blob_count(); }
-  size_t memory_available() const {
-    return kBlobStorageMaxMemoryUsage - memory_usage_;
-  }
 
   const BlobStorageRegistry& registry() { return registry_; }
 
- private:
-  using BlobRegistryEntry = BlobStorageRegistry::Entry;
-  using BlobConstructedCallback = BlobStorageRegistry::BlobConstructedCallback;
+  void EnableDisk(const base::FilePath& storage_directory,
+                  scoped_refptr<base::TaskRunner> file_runner);
+
+  // This builds a blob with the given |input_builder| and returns a handle to
+  // the constructed Blob. Blob metadata and data should be accessed through
+  // this handle.
+  // If there is data present that needs further population then we will call
+  // |can_populate_memory| when we're ready for the user data to be populated
+  // with the PENDING_DATA_POPULATION status. This can happen synchronously or
+  // asynchronously. Otherwise |can_populate_memory| should be null.
+  // In the further population case, the caller must call either
+  // FinishedPopulatingPendingBlob or BreakAndFinishPendingBlob after
+  // |can_populate_memory| is called to signify the data is finished populating
+  // or an error occurred (respectively).
+  // If the returned handle is broken, then the possible error cases are:
+  // * OUT_OF_MEMORY if we don't have enough memory to store the blob,
+  // * REFERENCED_BLOB_BROKEN if a referenced blob is broken or we're
+  //   referencing ourself.

[michaeln, 2016/08/15 22:44:44]

I'd vote to rename the following three methods (into two) and maybe delete some
comments for clarity.

CreatePendingBlob(...)
CompletePendingBlob(id, status);

BreakAndFinishPendingBlob vs FinishedPopulatingPendingBlob, those names have
toooo many terms in them to be easily recalled.

+  std::unique_ptr<BlobDataHandle> BuildBlob(
+      const BlobDataBuilder& input_builder,
+      const PopulatationAllowedCallback& population_allowed_callback);
+
+  // This breaks a blob that is currently being build by using the BuildBlob
+  // method above. Any callbacks waiting on this blob, including the
+  // |population_allowed_callback| callback given to BuildBlob, will be called
+  // with this status code.
+  void BreakAndFinishPendingBlob(const std::string& uuid, BlobStatus code);
+
+  // After calling BuildBlob above, the user should call this to notify the
+  // construction system that the unpopulated data in the given blob has been
+  // populated. Caller must have all pending items populated in the original
+  // builder |content| given in BuildBlob or we'll check-fail. If there isn't
+  // any pending data in the |input_builder| for the BuildBlob call, then this
+  // doesn't need to be called.
+  void FinishedPopulatingPendingBlob(const std::string& uuid);
+
+  const BlobMemoryController& memory_controller() { return memory_controller_; }
+
+ protected:
   friend class content::BlobDispatcherHost;
+  friend class content::BlobDispatcherHostTest;
   friend class BlobAsyncBuilderHost;
   friend class BlobAsyncBuilderHostTest;
   friend class BlobDataHandle;
   friend class BlobDataHandle::BlobDataHandleShared;
-  friend class BlobReaderTest;
-  FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, HandleBeforeAsyncCancel);
-  FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, ReadFromIncompleteBlob);
+  friend class BlobFlattenerTest;
   friend class BlobStorageContextTest;
-  FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, IncrementDecrementRef);
-  FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, OnCancelBuildingBlob);
-  FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, PublicBlobUrls);
-  FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest,
-                           TestUnknownBrokenAndBuildingBlobReference);
-  friend class ViewBlobInternalsJob;
-
-  // CompletePendingBlob or CancelPendingBlob should be called after this.
-  void CreatePendingBlob(const std::string& uuid,
-                         const std::string& content_type,
-                         const std::string& content_disposition);
-
-  // This includes resolving blob references in the builder. This will run the
-  // callbacks given in RunOnConstructionComplete.
-  void CompletePendingBlob(const BlobDataBuilder& external_builder);
-
-  // This will run the callbacks given in RunOnConstructionComplete.
-  void CancelPendingBlob(const std::string& uuid,
-                         IPCBlobCreationCancelCode reason);
 
   void IncrementBlobRefCount(const std::string& uuid);
   void DecrementBlobRefCount(const std::string& uuid);
 
-  // Methods called by BlobDataHandle:
   // This will return an empty snapshot until the blob is complete.
   // TODO(dmurph): After we make the snapshot method in BlobHandle private, then
   // make this DCHECK on the blob not being complete.
   std::unique_ptr<BlobDataSnapshot> CreateSnapshot(const std::string& uuid);
-  bool IsBroken(const std::string& uuid) const;
-  bool IsBeingBuilt(const std::string& uuid) const;
-  // Runs |done| when construction completes, with true if it was successful,
-  // and false if there was an error, which is reported in the second argument
-  // of the callback.
+
+  BlobStatus GetBlobStatus(const std::string& uuid) const;
+
+  // Runs |done| when construction completes with the final status of the blob.
   void RunOnConstructionComplete(const std::string& uuid,
-                                 const BlobConstructedCallback& done);
+                                 const BlobStatusCallback& done_callback);
 
-  // Appends the given blob item to the blob builder.  The new blob
-  // retains ownership of data_item if applicable, and returns false if there
-  // was an error and pouplates the error_code.  We can either have an error of:
-  // OUT_OF_MEMORY: We are out of memory to store this blob.
-  // REFERENCED_BLOB_BROKEN: One of the referenced blobs is broken.
-  bool AppendAllocatedBlobItem(const std::string& target_blob_uuid,
-                               scoped_refptr<BlobDataItem> data_item,
-                               InternalBlobData::Builder* target_blob_data,
-                               IPCBlobCreationCancelCode* error_code);
+  BlobStorageRegistry* mutable_registry() { return &registry_; }
 
-  // Allocates a shareable blob data item, with blob references resolved.  If
-  // there isn't enough memory, then a nullptr is returned.
-  scoped_refptr<ShareableBlobDataItem> AllocateShareableBlobDataItem(
-      const std::string& target_blob_uuid,
-      scoped_refptr<BlobDataItem> data_item);
+  BlobMemoryController* mutable_memory_controller() {
+    return &memory_controller_;
+  }
 
-  // Deconstructs the blob and appends it's contents to the target blob.  Items
-  // are shared if possible, and copied if the given offset and length
-  // have to split an item.
-  bool AppendBlob(const std::string& target_blob_uuid,
-                  const InternalBlobData& blob,
-                  uint64_t offset,
-                  uint64_t length,
-                  InternalBlobData::Builder* target_blob_data);
+ private:
+  std::unique_ptr<BlobDataHandle> CreateHandle(const std::string& uuid,
+                                               InternalBlobData* entry);
+
+  void FinishBuilding(InternalBlobData* entry);
+
+  // Callback given for our quota request for the copies data.
+  void OnEnoughSizeForCopies(
+      const std::string& uuid,
+      bool can_fit,
+      std::vector<BlobMemoryController::FileCreationInfo> files);
+
+  // Callback given for our quota request for user pending data.
+  void OnEnoughSizeForUserData(
+      const std::string& uuid,
+      bool can_fit,
+      std::vector<BlobMemoryController::FileCreationInfo> files);
+
+  void OnDependentBlobFinished(const std::string& owning_blob_uuid,
+                               BlobStatus reason);
+
+  void ClearAndFreeMemory(const std::string& uuid, InternalBlobData* entry);
+
+  // Shortcut method to set the status of the given items POPULATED_WITH_QUOTA.
+  // We expect the previous state to be QUOTA_GRANTED.
+  void SetItemStateToPopulated(std::vector<ShareableBlobDataItem*>* items);
 
   BlobStorageRegistry registry_;
-
-  // Used to keep track of how much memory is being utilized for blob data,
-  // we count only the items of TYPE_DATA which are held in memory and not
-  // items of TYPE_FILE.
-  size_t memory_usage_;
+  BlobMemoryController memory_controller_;
 
   DISALLOW_COPY_AND_ASSIGN(BlobStorageContext);
 };
 
 }  // namespace storage
 
 #endif  // STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_