[BlobAsync] Disk support for blob storage

storage/browser/blob/blob_storage_context.h

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_
 #define STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <map>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "base/callback_forward.h"
 #include "base/gtest_prod_util.h"
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
+#include "base/single_thread_task_runner.h"
 #include "storage/browser/blob/blob_data_handle.h"
+#include "storage/browser/blob/blob_memory_controller.h"
 #include "storage/browser/blob/blob_storage_registry.h"
 #include "storage/browser/blob/internal_blob_data.h"
 #include "storage/browser/storage_browser_export.h"
 #include "storage/common/blob_storage/blob_storage_constants.h"
 #include "storage/common/data_element.h"
 
 class GURL;
 
 namespace base {
 class FilePath;
@@ skipping 36 matching lines @@
       const BlobDataBuilder& builder);
 
   // Deprecated, use const ref version above.
   std::unique_ptr<BlobDataHandle> AddFinishedBlob(
       const BlobDataBuilder* builder);
 
   // Useful for coining blob urls from within the browser process.
   bool RegisterPublicBlobURL(const GURL& url, const std::string& uuid);
   void RevokePublicBlobURL(const GURL& url);
 
-  size_t memory_usage() const { return memory_usage_; }
   size_t blob_count() const { return registry_.blob_count(); }

[michaeln, 2016/07/07 20:05:21]

terminology things...
 finished  vs complete vs broken

We currently have these for BlobStates

  enum class BlobState {
    // The blob is pending transportation from the renderer. This is the default
    // state on entry construction.
    PENDING,
    // The blob is complete and can be read from.
    COMPLETE,
    // The blob is broken and no longer holds data. This happens when there was
    // a problem constructing the blob, or we've run out of memory.
    BROKEN
  };

I think this CL introduces a fourth state, something like

PENDING_MEMORY_ALLOCATION
PENDING_TRANSPORTATION
COMPLETE
BROKEN

I'd vote to codify that by adding to the enum.

-  size_t memory_available() const {
-    return kBlobStorageMaxMemoryUsage - memory_usage_;
+
+  const BlobStorageRegistry& registry() { return registry_; }
+
+  void EnableDisk(const base::FilePath storage_directory,

[michaeln, 2016/07/07 20:05:21]

const ref

+                  scoped_refptr<base::SingleThreadTaskRunner> file_runner);
+
+  std::unique_ptr<BlobDataHandle> BuildBrokenBlob(
+      const std::string& uuid,
+      const std::string& content_type,
+      const std::string& content_disposition,
+      IPCBlobCreationCancelCode reason);
+
+
+  std::unique_ptr<BlobDataHandle> BuildCompleteBlob(const BlobDataBuilder& content,

[michaeln, 2016/07/07 20:05:22]

|builder| for consistency here and below. in pre-existing methods that take a
builder as input, the input parameter is named 'builder'

+                                                    bool content_pending);
+
+  // We assume that any populated data can fit into our quota, where the called
+  // called BlobMemoryController::MaybeFitInMemoryNow and/or
+  // BlobMemoryController::NotifyWhenMemoryCanPopulated.
+  std::unique_ptr<BlobDataHandle> BuildBlob(const BlobDataBuilder& content,
+                                            bool content_pending);
+
+  void BreakAndFinishBlob(const std::string& uuid,
+                          IPCBlobCreationCancelCode code);
+
+  // This notifies the construction system that the unpopulated data in the
+  // given blob has been populated.
+  void FinishedPopulatingBlob(const std::string& uuid);

[michaeln, 2016/07/07 20:05:21]

terminology nits:

populating vs transporting
complete vs finished

i think it might be worth walking thru and being more consistent with naming

+
+ protected:
+  friend class BlobSlice;
+  friend class BlobFlattener;
+
+  friend class content::BlobDispatcherHostTest;
+
+  // Not thread safe.
+  static uint64_t GetAndIncrementItemId();
+
+  BlobMemoryController* mutable_memory_controller() {
+    return &memory_controller_;
   }
 
-  const BlobStorageRegistry& registry() { return registry_; }
+  const BlobMemoryController& memory_controller() { return memory_controller_; }
 
  private:
   using BlobRegistryEntry = BlobStorageRegistry::Entry;
   using BlobConstructedCallback = BlobStorageRegistry::BlobConstructedCallback;
   friend class content::BlobDispatcherHost;
   friend class BlobAsyncBuilderHost;
   friend class BlobAsyncBuilderHostTest;
   friend class BlobDataHandle;
   friend class BlobDataHandle::BlobDataHandleShared;
   friend class BlobReaderTest;
   FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, HandleBeforeAsyncCancel);
   FRIEND_TEST_ALL_PREFIXES(BlobReaderTest, ReadFromIncompleteBlob);
   friend class BlobStorageContextTest;
   FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, IncrementDecrementRef);
   FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, OnCancelBuildingBlob);
   FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest, PublicBlobUrls);
   FRIEND_TEST_ALL_PREFIXES(BlobStorageContextTest,
                            TestUnknownBrokenAndBuildingBlobReference);
   friend class ViewBlobInternalsJob;
 
-  // CompletePendingBlob or CancelPendingBlob should be called after this.
-  void CreatePendingBlob(const std::string& uuid,
-                         const std::string& content_type,
-                         const std::string& content_disposition);
+  std::unique_ptr<BlobDataHandle> CreateHandle(const std::string& uuid,
+                                               BlobRegistryEntry* entry);
 
-  // This includes resolving blob references in the builder. This will run the
-  // callbacks given in RunOnConstructionComplete.
-  void CompletePendingBlob(const BlobDataBuilder& external_builder);
+  bool CanFinishBuilding(BlobRegistryEntry* entry);
 
-  // This will run the callbacks given in RunOnConstructionComplete.
-  void CancelPendingBlob(const std::string& uuid,
-                         IPCBlobCreationCancelCode reason);
+  void FinishBuilding(BlobRegistryEntry* entry);
+
+  void OnEnoughSizeForDependentCopies(const std::string& uuid, bool can_fit);
+
+  void OnDependentBlobFinished(const std::string& owning_blob_uuid,
+                               bool construction_success,
+                               IPCBlobCreationCancelCode reason);
+
+  void ClearAndFreeMemory(const std::string& uuid, BlobRegistryEntry* entry);
 
   void IncrementBlobRefCount(const std::string& uuid);
   void DecrementBlobRefCount(const std::string& uuid);
 
   // Methods called by BlobDataHandle:
   // This will return an empty snapshot until the blob is complete.
   // TODO(dmurph): After we make the snapshot method in BlobHandle private, then
   // make this DCHECK on the blob not being complete.
   std::unique_ptr<BlobDataSnapshot> CreateSnapshot(const std::string& uuid);
   bool IsBroken(const std::string& uuid) const;
   bool IsBeingBuilt(const std::string& uuid) const;
   // Runs |done| when construction completes, with true if it was successful,
   // and false if there was an error, which is reported in the second argument
   // of the callback.
   void RunOnConstructionComplete(const std::string& uuid,
                                  const BlobConstructedCallback& done);
 
-  // Appends the given blob item to the blob builder.  The new blob
-  // retains ownership of data_item if applicable, and returns false if there
-  // was an error and pouplates the error_code.  We can either have an error of:
-  // OUT_OF_MEMORY: We are out of memory to store this blob.
-  // REFERENCED_BLOB_BROKEN: One of the referenced blobs is broken.
-  bool AppendAllocatedBlobItem(const std::string& target_blob_uuid,
-                               scoped_refptr<BlobDataItem> data_item,
-                               InternalBlobData::Builder* target_blob_data,
-                               IPCBlobCreationCancelCode* error_code);
+  void UpdateItemsInRecents(BlobRegistryEntry* entry);
 
-  // Allocates a shareable blob data item, with blob references resolved.  If
-  // there isn't enough memory, then a nullptr is returned.
-  scoped_refptr<ShareableBlobDataItem> AllocateShareableBlobDataItem(
-      const std::string& target_blob_uuid,
-      scoped_refptr<BlobDataItem> data_item);
-
-  // Deconstructs the blob and appends it's contents to the target blob.  Items
-  // are shared if possible, and copied if the given offset and length
-  // have to split an item.
-  bool AppendBlob(const std::string& target_blob_uuid,
-                  const InternalBlobData& blob,
-                  uint64_t offset,
-                  uint64_t length,
-                  InternalBlobData::Builder* target_blob_data);
+  void PerformCopy(const BlobStorageRegistry::ItemCopyEntry& copy,
+                   InternalBlobData* source,
+                   InternalBlobData* destination);
 
   BlobStorageRegistry registry_;
-
-  // Used to keep track of how much memory is being utilized for blob data,
-  // we count only the items of TYPE_DATA which are held in memory and not
-  // items of TYPE_FILE.
-  size_t memory_usage_;
+  BlobMemoryController memory_controller_;
 
   DISALLOW_COPY_AND_ASSIGN(BlobStorageContext);
 };
 
 }  // namespace storage
 
 #endif  // STORAGE_BROWSER_BLOB_BLOB_STORAGE_CONTEXT_H_