[BlobStorage] Enabling disk paging and direct storage.

content/browser/blob_storage/blob_memory_controller_unittest.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_memory_controller.h"
 
 #include "base/bind.h"
 #include "base/files/file_util.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/message_loop/message_loop.h"
@@ skipping 16 matching lines @@
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
 
 const std::string kBlobStorageDirectory = "blob_storage";
 const size_t kTestBlobStorageIPCThresholdBytes = 20;
 const size_t kTestBlobStorageMaxSharedMemoryBytes = 50;
 const size_t kTestBlobStorageMaxBlobMemorySize = 500;
 const uint64_t kTestBlobStorageMaxDiskSpace = 1000;
 const uint64_t kTestBlobStorageMinFileSizeBytes = 10;
 const uint64_t kTestBlobStorageMaxFileSizeBytes = 100;
 
+const uint64_t kTestSmallBlobStorageMaxDiskSpace = 100;
+
 class BlobMemoryControllerTest : public testing::Test {
  protected:
   BlobMemoryControllerTest() {}
 
   void SetUp() override {
     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
     base::ThreadRestrictions::SetIOAllowed(false);
   };
 
   void TearDown() override {
@@ skipping 20 matching lines @@
     BlobStorageLimits limits;
     limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
     limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
     limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
     limits.max_blob_disk_space = kTestBlobStorageMaxDiskSpace;
     limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
     limits.max_file_size = kTestBlobStorageMaxFileSizeBytes;
     controller->set_limits_for_testing(limits);
   }
 
+  void SetSmallDiskTestMemoryLimits(BlobMemoryController* controller) {
+    BlobStorageLimits limits;
+    limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
+    limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
+    limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
+    limits.max_blob_disk_space = kTestSmallBlobStorageMaxDiskSpace;
+    limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
+    limits.max_file_size = kTestBlobStorageMaxFileSizeBytes;
+    controller->set_limits_for_testing(limits);
+  }
+
   void SaveFileCreationInfo(std::vector<FileCreationInfo> info, bool success) {
     file_quota_result_ = success;
     if (success) {
       files_created_.swap(info);
     }
   }
 
+  void SaveMemoryRequestToOutput(bool* output, bool success) {
+    ASSERT_TRUE(output);
+    *output = success;
+  }
   void SaveMemoryRequest(bool success) { memory_quota_result_ = success; }
 
   BlobMemoryController::FileQuotaRequestCallback GetFileCreationCallback() {
     return base::Bind(&BlobMemoryControllerTest::SaveFileCreationInfo,
                       base::Unretained(this));
   }
 
   BlobMemoryController::MemoryQuotaRequestCallback GetMemoryRequestCallback() {
     return base::Bind(&BlobMemoryControllerTest::SaveMemoryRequest,
                       base::Unretained(this));
   }
 
+  BlobMemoryController::MemoryQuotaRequestCallback
+  GetMemoryRequestCallbackToOutput(bool* output) {
+    return base::Bind(&BlobMemoryControllerTest::SaveMemoryRequestToOutput,
+                      base::Unretained(this), output);
+  }
+
   void RunFileThreadTasks() {
     base::ThreadRestrictions::SetIOAllowed(true);
     file_runner_->RunPendingTasks();
     base::ThreadRestrictions::SetIOAllowed(false);
   }
 
   bool HasMemoryAllocation(ShareableBlobDataItem* item) {
     return static_cast<bool>(item->memory_allocation_);
   }
 
@@ skipping 46 matching lines @@
                                   0, kTestBlobStorageMaxBlobMemorySize -
                                          kTestBlobStorageMinFileSizeBytes + 1));
 
     EXPECT_EQ(Strategy::FILE, controller.DetermineStrategy(
                                   0, kTestBlobStorageMaxBlobMemorySize));
 
     // Too large for disk.
     EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(
                                        0, kTestBlobStorageMaxDiskSpace + 1));
   }
+  {
+    BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+    SetSmallDiskTestMemoryLimits(&controller);
+
+    EXPECT_TRUE(controller.CanReserveQuota(kTestBlobStorageMaxBlobMemorySize));
+    // Since our disk is too small, this should be sent with shared memory.
+    EXPECT_EQ(
+        Strategy::SHARED_MEMORY,
+        controller.DetermineStrategy(0, kTestBlobStorageMaxBlobMemorySize));
+  }
 }
 
 TEST_F(BlobMemoryControllerTest, GrantMemory) {
   const std::string kId = "id";
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
   SetTestMemoryLimits(&controller);
 
   BlobDataBuilder builder(kId);
   builder.AppendFutureData(10);
   builder.AppendFutureData(20);
@@ skipping 102 matching lines @@
   items2.clear();
   EXPECT_EQ(0u, controller.memory_usage());
   items.clear();
   EXPECT_TRUE(file_runner_->HasPendingTask());
   RunFileThreadTasks();
   base::RunLoop().RunUntilIdle();
   EXPECT_EQ(0u, controller.disk_usage());
 }
 
 TEST_F(BlobMemoryControllerTest, NoDiskTooLarge) {
-  const std::string kId = "id";
   BlobMemoryController controller(temp_dir_.GetPath(), nullptr);
   SetTestMemoryLimits(&controller);
 
   EXPECT_FALSE(controller.CanReserveQuota(kTestBlobStorageMaxBlobMemorySize +
                                           kTestBlobStorageMinFileSizeBytes +
                                           1));
 }
 
 TEST_F(BlobMemoryControllerTest, TooLargeForDisk) {
-  const std::string kId = "id";
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
   SetTestMemoryLimits(&controller);
 
   EXPECT_FALSE(controller.CanReserveQuota(kTestBlobStorageMaxDiskSpace + 1));
 }
 
 TEST_F(BlobMemoryControllerTest, CancelMemoryRequest) {
   const std::string kId = "id";
   const std::string kId2 = "id2";
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
@@ skipping 282 matching lines @@
   EXPECT_EQ(
       kTestBlobStorageMaxBlobMemorySize - kTestBlobStorageMinFileSizeBytes,
       controller.memory_usage());
   EXPECT_EQ(
       kTestBlobStorageMaxBlobMemorySize - kTestBlobStorageMinFileSizeBytes,
       controller.disk_usage());
 
   EXPECT_TRUE(memory_quota_result_);
 }
 
+TEST_F(BlobMemoryControllerTest, PagingStopsWhenFull) {
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+  const size_t kTotalBlobStorageSize =
+      kTestBlobStorageMaxDiskSpace + kTestBlobStorageMaxBlobMemorySize;
+
+  const size_t kDataSize = 10u;
+  const size_t kBlobsThatCanFit = kTotalBlobStorageSize / kDataSize;
+  const size_t kNumFastBlobs = kTestBlobStorageMaxBlobMemorySize / kDataSize;
+  char kData[10];
+  memset(kData, 'e', kDataSize);
+
+  // Create all of our blobs.
+  std::vector<scoped_refptr<ShareableBlobDataItem>> all_items;
+  std::vector<base::WeakPtr<QuotaAllocationTask>> memory_tasks;
+  bool memory_requested[kBlobsThatCanFit] = {};
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    BlobDataBuilder builder("fake");
+    builder.AppendData(kData, kDataSize);
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(builder);
+    EXPECT_TRUE(controller.CanReserveQuota(kDataSize));
+    EXPECT_EQ((i < kNumFastBlobs) ? Strategy::NONE_NEEDED : Strategy::IPC,
+              controller.DetermineStrategy(kDataSize, kDataSize))
+        << i;
+    base::WeakPtr<QuotaAllocationTask> memory_task =
+        controller.ReserveMemoryQuota(
+            items, GetMemoryRequestCallbackToOutput(memory_requested + i));
+    if (memory_task) {
+      memory_tasks.push_back(std::move(memory_task));
+    }
+    all_items.insert(all_items.end(), items.begin(), items.end());
+  }
+  // We should have stored all of our memory quota, and no disk yet.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(0ull, controller.disk_usage());
+
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_EQ(i < kBlobsThatCanFit / 3, memory_requested[i]) << i;
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      EXPECT_TRUE(memory_requested[i]);
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+
+  // This will schedule one task. Paging starts as soon as there is enough
+  // memory to page, and multiple pagings can't happen at the same time.
+  EXPECT_EQ(10ull, controller.disk_usage());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // The rest of the tasks should be scheduled.
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // Everything in memory should be on disk, and next batch of memory items
+  // should be granted.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(500ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+
+  // Flag next batch for saving to disk.
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_EQ(i < kBlobsThatCanFit * 2 / 3, memory_requested[i]) << i;
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+
+  // Same as before. One page task is scheduled, so run them twice.
+  EXPECT_EQ(510ull, controller.disk_usage());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // We page one time first, as it blocks paging once it starts.
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+
+  // All quota should be allocated.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(1000ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+
+  // Flag last batch as populated.
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_TRUE(memory_requested[i]);
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+
+  // There should be no more paging to disk, as we've reached the end.
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+
+  // All quota should be allocated still.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(1000ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+}
+
 TEST_F(BlobMemoryControllerTest, DisableDiskWithFileAndMemoryPending) {
   const std::string kFirstMemoryId = "id";
   const uint64_t kFirstMemorySize = kTestBlobStorageMaxBlobMemorySize;
   const std::string kSecondMemoryId = "id2";
   const uint64_t kSecondMemorySize = 1;
   const std::string kFileId = "id2";
   const uint64_t kFileBlobSize = kTestBlobStorageMaxBlobMemorySize;
 
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
   SetTestMemoryLimits(&controller);
@@ skipping 64 matching lines @@
   file_items.clear();
   items.clear();
 
   RunFileThreadTasks();
   base::RunLoop().RunUntilIdle();
 
   EXPECT_EQ(0ull, controller.disk_usage());
   EXPECT_EQ(0ull, controller.memory_usage());
 }
 }  // namespace storage