[BlobAsync] Disk support for blob storage

content/browser/blob_storage/blob_storage_context_unittest.cc

Index: content/browser/blob_storage/blob_storage_context_unittest.cc
diff --git a/content/browser/blob_storage/blob_storage_context_unittest.cc b/content/browser/blob_storage/blob_storage_context_unittest.cc
index e03d5f3b74a329c7a8a1f0caa399a034c9cf1ea5..b4c5e33d3f7ac398711044c59929515c991d171c 100644
--- a/content/browser/blob_storage/blob_storage_context_unittest.cc
+++ b/content/browser/blob_storage/blob_storage_context_unittest.cc
@@ -10,11 +10,16 @@
 #include <memory>
 #include <string>
 
+#include "base/bind.h"
 #include "base/files/file.h"
 #include "base/files/file_path.h"
+#include "base/files/file_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/message_loop/message_loop.h"
 #include "base/run_loop.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/test/test_simple_task_runner.h"
+#include "base/threading/thread_task_runner_handle.h"
 #include "base/time/time.h"
 #include "content/browser/blob_storage/blob_dispatcher_host.h"
 #include "content/browser/blob_storage/chrome_blob_storage_context.h"
@@ -27,7 +32,6 @@
 #include "storage/browser/blob/blob_data_handle.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/blob_data_snapshot.h"
-#include "storage/browser/blob/blob_transport_result.h"
 #include "storage/common/blob_storage/blob_item_bytes_request.h"
 #include "storage/common/blob_storage/blob_item_bytes_response.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ -37,11 +41,21 @@ using RequestMemoryCallback =
 
 namespace storage {
 namespace {
+using base::TestSimpleTaskRunner;
 
-const char kContentType[] = "text/plain";
-const char kContentDisposition[] = "content_disposition";
 const int kTestDiskCacheStreamIndex = 0;
 
+const std::string kBlobStorageDirectory = "blob_storage";
+const size_t kTestBlobStorageIPCThresholdBytes = 20;
+const size_t kTestBlobStorageMaxSharedMemoryBytes = 50;
+
+const size_t kTestBlobStorageMaxBlobMemorySize = 400;
+const size_t kTestBlobStorageMaxMemoryUsage = 500;
+const uint64_t kTestBlobStorageMaxDiskSpace = 4000;
+const size_t kTestBlobStorageInFlightMemory = 10;
+const uint64_t kTestBlobStorageMinFileSizeBytes = 10;
+const uint64_t kTestBlobStorageMaxFileSizeBytes = 100;
+
 // Our disk cache tests don't need a real data handle since the tests themselves
 // scope the disk cache and entries.
 class EmptyDataHandle : public storage::BlobDataBuilder::DataHandle {
@@ -79,6 +93,14 @@ disk_cache::ScopedEntryPtr CreateDiskCacheEntry(disk_cache::Backend* cache,
   return entry;
 }
 
+void SaveBlobStatus(BlobStatus* status_ptr, BlobStatus status) {
+  *status_ptr = status;
+}
+
+void IncrementPointer(size_t* number, BlobStatus status) {
+  EXPECT_EQ(BlobStatus::DONE, status);
+  *number = *number + 1;
+}
 
 }  // namespace
 
@@ -87,6 +109,13 @@ class BlobStorageContextTest : public testing::Test {
   BlobStorageContextTest() {}
   ~BlobStorageContextTest() override {}
 
+  void TearDown() override {
+    // Make sure we clean up the files.
+    base::RunLoop().RunUntilIdle();
+    file_runner_->RunPendingTasks();
+    ASSERT_EQ(true, base::DeleteFile(temp_dir_, true));
+  }
+
   std::unique_ptr<BlobDataHandle> SetupBasicBlob(const std::string& id) {
     BlobDataBuilder builder(id);
     builder.AppendData("1", 1);
@@ -94,12 +123,220 @@ class BlobStorageContextTest : public testing::Test {
     return context_.AddFinishedBlob(builder);
   }
 
+  void SetTestMemoryLimits() {
+    context_.mutable_memory_controller()->SetMemoryConstantsForTesting(
+        kTestBlobStorageIPCThresholdBytes, kTestBlobStorageMaxSharedMemoryBytes,
+        kTestBlobStorageMaxBlobMemorySize, kTestBlobStorageMaxMemoryUsage,
+        kTestBlobStorageMaxDiskSpace, kTestBlobStorageInFlightMemory,
+        kTestBlobStorageMinFileSizeBytes, kTestBlobStorageMaxFileSizeBytes);
+  }
+
+  base::FilePath temp_dir_;
+  scoped_refptr<TestSimpleTaskRunner> file_runner_ = new TestSimpleTaskRunner();
+
+  base::MessageLoop fake_io_message_loop;
   BlobStorageContext context_;
 };
 
-TEST_F(BlobStorageContextTest, IncrementDecrementRef) {
-  base::MessageLoop fake_io_message_loop;
+TEST_F(BlobStorageContextTest, BuildBlobAsync) {
+  const std::string kId("id");
+  const size_t kSize = 10u;
+  BlobStatus status = BlobStatus::INVALID_CONSTRUCTION_ARGUMENTS;
+
+  BlobDataBuilder builder(kId);
+  builder.AppendFutureData(kSize);
+  builder.set_content_type("text/plain");
+  EXPECT_EQ(0lu, context_.memory_controller().memory_usage());
+  std::unique_ptr<BlobDataHandle> handle =
+      context_.BuildBlob(builder, base::Bind(&SaveBlobStatus, &status));
+  EXPECT_EQ(10lu, context_.memory_controller().memory_usage());
+  EXPECT_TRUE(handle->IsBeingBuilt());
+  EXPECT_EQ(BlobStatus::PENDING, status);
+
+  BlobStatus construction_done = BlobStatus::INVALID_CONSTRUCTION_ARGUMENTS;
+  handle->RunOnConstructionComplete(
+      base::Bind(&SaveBlobStatus, &construction_done));
+
+  EXPECT_EQ(10u, context_.memory_controller().memory_usage());
+
+  builder.PopulateFutureData(0, "abcdefghij", 0, 10u);
+  context_.FinishedPopulatingBlob(kId);
+
+  // Check we're done.
+  EXPECT_EQ(BlobStatus::DONE, context_.GetBlobStatus(kId));
+  base::RunLoop().RunUntilIdle();
+  EXPECT_EQ(BlobStatus::DONE, construction_done);
 
+  EXPECT_EQ(builder, *context_.CreateSnapshot(kId));
+
+  handle.reset();
+  base::RunLoop().RunUntilIdle();
+  EXPECT_EQ(0lu, context_.memory_controller().memory_usage());
+}
+
+TEST_F(BlobStorageContextTest, BuildBlobAndCancel) {
+  const std::string kId("id");
+  const size_t kSize = 10u;
+  BlobStatus status = BlobStatus::INVALID_CONSTRUCTION_ARGUMENTS;
+
+  BlobDataBuilder builder(kId);
+  builder.AppendFutureData(kSize);
+  builder.set_content_type("text/plain");
+  EXPECT_EQ(0lu, context_.memory_controller().memory_usage());
+  std::unique_ptr<BlobDataHandle> handle =
+      context_.BuildBlob(builder, base::Bind(&SaveBlobStatus, &status));
+  EXPECT_EQ(10lu, context_.memory_controller().memory_usage());
+  EXPECT_TRUE(handle->IsBeingBuilt());
+  EXPECT_EQ(BlobStatus::PENDING, status);
+  EXPECT_EQ(10u, context_.memory_controller().memory_usage());
+
+  BlobStatus construction_done = BlobStatus::INVALID_CONSTRUCTION_ARGUMENTS;
+  handle->RunOnConstructionComplete(
+      base::Bind(&SaveBlobStatus, &construction_done));
+
+  context_.BreakAndFinishBlob(kId, BlobStatus::SOURCE_DIED_IN_TRANSIT);
+  EXPECT_TRUE(handle->IsBroken());
+  EXPECT_EQ(0lu, context_.memory_controller().memory_usage());
+
+  // Check we're broken.
+  EXPECT_EQ(BlobStatus::SOURCE_DIED_IN_TRANSIT, context_.GetBlobStatus(kId));
+  base::RunLoop().RunUntilIdle();
+  EXPECT_EQ(BlobStatus::SOURCE_DIED_IN_TRANSIT, construction_done);
+}
+
+TEST_F(BlobStorageContextTest, BuildBlobFuzzish) {
+  const std::string kId("id");
+  const size_t kTotalRawBlobs = 200;
+  const size_t kTotalSlicedBlobs = 100;
+  SetTestMemoryLimits();
+
+  // This tests mixed blob content, both async and synchronous content, and
+
+  std::vector<std::unique_ptr<BlobDataBuilder>> builders;
+  std::vector<size_t> sizes;
+
+  for (size_t i = 0; i < kTotalRawBlobs; i++) {
+    builders.emplace_back(new BlobDataBuilder(base::SizeTToString(i)));
+    auto& builder = *builders.back();
+    size_t size = 0;
+    if (i % 2 != 0) {
+      builder.AppendFutureData(5u);
+      size += 5u;
+    }
+    if (i % 5 != 0) {
+      builder.AppendFile(base::FilePath(base::SizeTToString(i)), 0ul, 20ul,
+                         base::Time::Max());
+      size += 20u;
+    }
+    if (i % 3 != 0) {
+      builder.AppendData("abcdefghij", 4u);
+      size += 4u;
+    }
+    if (i % 3 == 0) {
+      builder.AppendFutureData(1u);
+      size += 1u;
+    }
+    if (i % 7 == 0) {
+      builder.AppendFutureFile(0lu, 3lu);
+      size += 3u;
+    }
+    sizes.push_back(size);
+  }
+
+  for (size_t i = 0; i < kTotalSlicedBlobs; i++) {
+    builders.emplace_back(
+        new BlobDataBuilder(base::SizeTToString(i + kTotalRawBlobs)));
+    size_t source_size = sizes[i];
+    size_t offset = sizes[i] == 1 ? 0 : i % (source_size - 1);
+    size_t size = (i % (source_size - offset)) + 1;
+    builders.back()->AppendBlob(base::SizeTToString(i), offset, size);
+    sizes.push_back(size);
+  }
+
+  size_t total_finished_blobs = 0;
+  std::vector<std::unique_ptr<BlobDataHandle>> handles;
+  std::vector<BlobStatus> statuses;
+  std::vector<bool> populated;
+  statuses.resize(kTotalRawBlobs, BlobStatus::INVALID_CONSTRUCTION_ARGUMENTS);
+  populated.resize(kTotalRawBlobs, false);
+  context_.EnableDisk(temp_dir_, file_runner_);
+
+  for (size_t i = 0; i < builders.size(); i++) {
+    BlobDataBuilder& builder = *builders[i];
+    builder.set_content_type("text/plain");
+    bool has_pending_memory =
+        i < kTotalRawBlobs && (i % 3 == 0 || i % 2 != 0 || i % 7 == 0);
+    std::unique_ptr<BlobDataHandle> handle = context_.BuildBlob(
+        builder, has_pending_memory
+                     ? base::Bind(&SaveBlobStatus, &statuses[0] + i)
+                     : BlobStatusCallback());
+    handle->RunOnConstructionComplete(
+        base::Bind(&IncrementPointer, &total_finished_blobs));
+    handles.push_back(std::move(handle));
+  }
+  base::RunLoop().RunUntilIdle();
+
+  // We should be needing to send a page or two to disk.
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+
+  do {
+    file_runner_->RunPendingTasks();
+    base::RunLoop().RunUntilIdle();
+
+    // Continue populating data for items that can fit.
+    for (size_t i = 0; i < kTotalRawBlobs; i++) {
+      auto& builder = *builders[i];
+      bool has_pending_memory = i % 3 == 0 || i % 2 != 0 || i % 7 == 0;
+      if (has_pending_memory && !populated[i] &&
+          statuses[i] == BlobStatus::PENDING) {
+        if (i % 2 != 0) {
+          builder.PopulateFutureData(0, "abcde", 0, 5);
+        }
+        if (i % 3 == 0) {
+          size_t index = i % 7 == 0 ? builder.items_.size() - 2
+                                    : builder.items_.size() - 1;
+          builder.PopulateFutureData(index, "z", 0, 1);
+        }
+        if (i % 7 == 0) {
+          scoped_refptr<ShareableFileReference> file_ref =
+              ShareableFileReference::GetOrCreate(
+                  base::FilePath(base::SizeTToString(i + kTotalRawBlobs)),
+                  ShareableFileReference::DONT_DELETE_ON_FINAL_RELEASE,
+                  file_runner_.get());
+          builder.PopulateFutureFile(builder.items_.size() - 1, file_ref,
+                                     base::Time::Max());
+        }
+        context_.FinishedPopulatingBlob(base::SizeTToString(i));
+        populated[i] = true;
+      }
+    }
+    base::RunLoop().RunUntilIdle();
+  } while (file_runner_->HasPendingTask());
+
+  // We should be completely built now.
+  EXPECT_EQ(kTotalRawBlobs + kTotalSlicedBlobs, total_finished_blobs);
+
+  handles.clear();
+  base::RunLoop().RunUntilIdle();
+  // We should have file cleanup tasks.
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  file_runner_->RunPendingTasks();
+
+  for (size_t i = 0; i < kTotalRawBlobs; i++) {
+    bool has_pending_memory = i % 3 == 0 || i % 2 != 0;
+    if (has_pending_memory)
+      EXPECT_EQ(BlobStatus::PENDING, statuses[i]) << i;
+  }
+  EXPECT_EQ(0lu, context_.memory_controller().memory_usage());
+  EXPECT_EQ(0lu, context_.memory_controller().disk_usage());
+}
+
+TEST_F(BlobStorageContextTest, TestErrors) {
+  // bad slice offset & size
+  // Finish before finished
+}
+
+TEST_F(BlobStorageContextTest, IncrementDecrementRef) {
   // Build up a basic blob.
   const std::string kId("id");
   std::unique_ptr<BlobDataHandle> blob_data_handle = SetupBasicBlob(kId);
@@ -122,23 +359,7 @@ TEST_F(BlobStorageContextTest, IncrementDecrementRef) {
   EXPECT_FALSE(blob_data_handle);
 }
 
-TEST_F(BlobStorageContextTest, OnCancelBuildingBlob) {
-  base::MessageLoop fake_io_message_loop;
-
-  // Build up a basic blob.
-  const std::string kId("id");
-  context_.CreatePendingBlob(kId, std::string(kContentType),
-                             std::string(kContentDisposition));
-  EXPECT_TRUE(context_.IsBeingBuilt(kId));
-  context_.CancelPendingBlob(kId, IPCBlobCreationCancelCode::OUT_OF_MEMORY);
-  EXPECT_TRUE(context_.registry().HasEntry(kId));
-  EXPECT_FALSE(context_.IsBeingBuilt(kId));
-  EXPECT_TRUE(context_.IsBroken(kId));
-}
-
 TEST_F(BlobStorageContextTest, BlobDataHandle) {
-  base::MessageLoop fake_io_message_loop;
-
   // Build up a basic blob.
   const std::string kId("id");
   std::unique_ptr<BlobDataHandle> blob_data_handle = SetupBasicBlob(kId);
@@ -166,8 +387,6 @@ TEST_F(BlobStorageContextTest, MemoryUsage) {
   const std::string kId1("id1");
   const std::string kId2("id2");
 
-  base::MessageLoop fake_io_message_loop;
-
   BlobDataBuilder builder1(kId1);
   BlobDataBuilder builder2(kId2);
   builder1.AppendData("Data1Data2");
@@ -179,26 +398,26 @@ TEST_F(BlobStorageContextTest, MemoryUsage) {
   builder2.AppendBlob(kId1);
   builder2.AppendBlob(kId1);
 
-  EXPECT_EQ(0lu, context_.memory_usage());
+  EXPECT_EQ(0lu, context_.memory_controller().memory_usage());
 
   std::unique_ptr<BlobDataHandle> blob_data_handle =
       context_.AddFinishedBlob(&builder1);
-  EXPECT_EQ(10lu, context_.memory_usage());
+  EXPECT_EQ(10lu, context_.memory_controller().memory_usage());
   std::unique_ptr<BlobDataHandle> blob_data_handle2 =
       context_.AddFinishedBlob(&builder2);
-  EXPECT_EQ(10lu, context_.memory_usage());
+  EXPECT_EQ(10lu, context_.memory_controller().memory_usage());
 
   EXPECT_EQ(2u, context_.registry().blob_count());
 
   blob_data_handle.reset();
   base::RunLoop().RunUntilIdle();
 
-  EXPECT_EQ(10lu, context_.memory_usage());
+  EXPECT_EQ(10lu, context_.memory_controller().memory_usage());
   EXPECT_EQ(1u, context_.registry().blob_count());
   blob_data_handle2.reset();
   base::RunLoop().RunUntilIdle();
 
-  EXPECT_EQ(0lu, context_.memory_usage());
+  EXPECT_EQ(0lu, context_.memory_controller().memory_usage());
   EXPECT_EQ(0u, context_.registry().blob_count());
 }
 
@@ -209,8 +428,6 @@ TEST_F(BlobStorageContextTest, AddFinishedBlob) {
   const std::string kId3("id3");
   const std::string kId3Prime("id3.prime");
 
-  base::MessageLoop fake_io_message_loop;
-
   BlobDataBuilder builder1(kId1);
   BlobDataBuilder builder2(kId2);
   BlobDataBuilder canonicalized_blob_data2(kId2Prime);
@@ -275,12 +492,11 @@ TEST_F(BlobStorageContextTest, AddFinishedBlob_LargeOffset) {
   // A value which does not fit in a 4-byte data type. Used to confirm that
   // large values are supported on 32-bit Chromium builds. Regression test for:
   // crbug.com/458122.
-  const uint64_t kLargeSize = std::numeric_limits<uint64_t>::max();
+  const uint64_t kLargeSize = std::numeric_limits<uint64_t>::max() - 1;
 
   const uint64_t kBlobLength = 5;
   const std::string kId1("id1");
   const std::string kId2("id2");
-  base::MessageLoop fake_io_message_loop;
 
   BlobDataBuilder builder1(kId1);
   builder1.AppendFileSystemFile(GURL(), 0, kLargeSize, base::Time::Now());
@@ -307,7 +523,6 @@ TEST_F(BlobStorageContextTest, AddFinishedBlob_LargeOffset) {
 }
 
 TEST_F(BlobStorageContextTest, BuildDiskCacheBlob) {
-  base::MessageLoop fake_io_message_loop;
   scoped_refptr<BlobDataBuilder::DataHandle>
       data_handle = new EmptyDataHandle();
 
@@ -349,8 +564,6 @@ TEST_F(BlobStorageContextTest, CompoundBlobs) {
   const std::string kId3("id3");
   const std::string kId2Prime("id2.prime");
 
-  base::MessageLoop fake_io_message_loop;
-
   // Setup a set of blob data for testing.
   base::Time time1, time2;
   base::Time::FromString("Tue, 15 Nov 1994, 12:45:26 GMT", &time1);
@@ -414,8 +627,6 @@ TEST_F(BlobStorageContextTest, CompoundBlobs) {
 }
 
 TEST_F(BlobStorageContextTest, PublicBlobUrls) {
-  base::MessageLoop fake_io_message_loop;
-
   // Build up a basic blob.
   const std::string kId("id");
   std::unique_ptr<BlobDataHandle> first_handle = SetupBasicBlob(kId);
@@ -437,8 +648,8 @@ TEST_F(BlobStorageContextTest, PublicBlobUrls) {
   blob_data_handle = context_.GetBlobDataFromPublicURL(kUrl);
   EXPECT_TRUE(blob_data_handle);
   blob_data_handle.reset();
-  base::RunLoop().RunUntilIdle();
 
+  base::RunLoop().RunUntilIdle();
   // Finally get rid of the url registration and the blob.
   context_.RevokePublicBlobURL(kUrl);
   blob_data_handle = context_.GetBlobDataFromPublicURL(kUrl);
@@ -447,17 +658,15 @@ TEST_F(BlobStorageContextTest, PublicBlobUrls) {
 }
 
 TEST_F(BlobStorageContextTest, TestUnknownBrokenAndBuildingBlobReference) {
-  base::MessageLoop fake_io_message_loop;
   const std::string kBrokenId("broken_id");
   const std::string kBuildingId("building_id");
   const std::string kReferencingId("referencing_id");
   const std::string kUnknownId("unknown_id");
 
-  // Create a broken blob and a building blob.
-  context_.CreatePendingBlob(kBuildingId, "", "");
-  context_.CreatePendingBlob(kBrokenId, "", "");
-  context_.CancelPendingBlob(kBrokenId, IPCBlobCreationCancelCode::UNKNOWN);
-  EXPECT_TRUE(context_.IsBroken(kBrokenId));
+  // Create a broken blob.
+  std::unique_ptr<BlobDataHandle> broken_handle =
+      context_.BuildBrokenBlob(kBrokenId, "", "", BlobStatus::OUT_OF_MEMORY);
+  EXPECT_TRUE(context_.GetBlobStatus(kBrokenId) == BlobStatus::OUT_OF_MEMORY);
   EXPECT_TRUE(context_.registry().HasEntry(kBrokenId));
 
   // Try to create a blob with a reference to an unknown blob.