[BlobStorage] Enabling disk paging and direct storage.

content/browser/blob_storage/blob_memory_controller_unittest.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "storage/browser/blob/blob_memory_controller.h"
 
 #include "base/bind.h"
 #include "base/files/file_util.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/message_loop/message_loop.h"
 #include "base/run_loop.h"
 #include "base/test/test_simple_task_runner.h"
 #include "base/threading/thread_restrictions.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "storage/browser/blob/blob_data_builder.h"
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
 #include "storage/common/data_element.h"
+#include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace storage {
 
 using Strategy = BlobMemoryController::Strategy;
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using base::TestSimpleTaskRunner;
 using ItemState = ShareableBlobDataItem::State;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+using ::testing::Return;
 
 const std::string kBlobStorageDirectory = "blob_storage";
 const size_t kTestBlobStorageIPCThresholdBytes = 20;
 const size_t kTestBlobStorageMaxSharedMemoryBytes = 50;
 const size_t kTestBlobStorageMaxBlobMemorySize = 500;
 const uint64_t kTestBlobStorageMaxDiskSpace = 1000;
 const uint64_t kTestBlobStorageMinFileSizeBytes = 10;
 const uint64_t kTestBlobStorageMaxFileSizeBytes = 100;
 
+const uint64_t kTestSmallBlobStorageMaxDiskSpace = 100;
+
+class MockDiskSpaceTestGetter
+    : public BlobMemoryController::DiskSpaceTestGetter {
+ public:
+  MockDiskSpaceTestGetter() {}
+  virtual ~MockDiskSpaceTestGetter() {}
+
+  MOCK_CONST_METHOD0(AmountOfFreeDiskSpace, int64_t());
+};
+
 class BlobMemoryControllerTest : public testing::Test {
  protected:
   BlobMemoryControllerTest() {}
 
   void SetUp() override {
     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
     base::ThreadRestrictions::SetIOAllowed(false);
   };
 
   void TearDown() override {
@@ skipping 14 matching lines @@
           builder.items_[i], ShareableBlobDataItem::QUOTA_NEEDED)));
     }
     return result;
   }
 
   void SetTestMemoryLimits(BlobMemoryController* controller) {
     BlobStorageLimits limits;
     limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
     limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
     limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
-    limits.max_blob_disk_space = kTestBlobStorageMaxDiskSpace;
+    limits.desired_max_disk = kTestBlobStorageMaxDiskSpace;
+    limits.effective_max_disk = kTestBlobStorageMaxDiskSpace;
     limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
     limits.max_file_size = kTestBlobStorageMaxFileSizeBytes;
     controller->set_limits_for_testing(limits);
+  }
+
+  void SetSmallDiskTestMemoryLimits(BlobMemoryController* controller) {
+    BlobStorageLimits limits;
+    limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
+    limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
+    limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
+    limits.desired_max_disk = kTestSmallBlobStorageMaxDiskSpace;
+    limits.effective_max_disk = kTestSmallBlobStorageMaxDiskSpace;
+    limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
+    limits.max_file_size = kTestBlobStorageMaxFileSizeBytes;
+    controller->set_limits_for_testing(limits);
   }
 
   void SaveFileCreationInfo(std::vector<FileCreationInfo> info, bool success) {
     file_quota_result_ = success;
     if (success) {
       files_created_.swap(info);
     }
   }
 
+  void SaveMemoryRequestToOutput(bool* output, bool success) {
+    ASSERT_TRUE(output);
+    *output = success;
+  }
   void SaveMemoryRequest(bool success) { memory_quota_result_ = success; }
 
   BlobMemoryController::FileQuotaRequestCallback GetFileCreationCallback() {
     return base::Bind(&BlobMemoryControllerTest::SaveFileCreationInfo,
                       base::Unretained(this));
   }
 
   BlobMemoryController::MemoryQuotaRequestCallback GetMemoryRequestCallback() {
     return base::Bind(&BlobMemoryControllerTest::SaveMemoryRequest,
                       base::Unretained(this));
   }
 
+  BlobMemoryController::MemoryQuotaRequestCallback
+  GetMemoryRequestCallbackToOutput(bool* output) {
+    return base::Bind(&BlobMemoryControllerTest::SaveMemoryRequestToOutput,
+                      base::Unretained(this), output);
+  }
+
   void RunFileThreadTasks() {
     base::ThreadRestrictions::SetIOAllowed(true);
     file_runner_->RunPendingTasks();
     base::ThreadRestrictions::SetIOAllowed(false);
   }
 
   bool HasMemoryAllocation(ShareableBlobDataItem* item) {
     return static_cast<bool>(item->memory_allocation_);
   }
 
@@ skipping 46 matching lines @@
                                   0, kTestBlobStorageMaxBlobMemorySize -
                                          kTestBlobStorageMinFileSizeBytes + 1));
 
     EXPECT_EQ(Strategy::FILE, controller.DetermineStrategy(
                                   0, kTestBlobStorageMaxBlobMemorySize));
 
     // Too large for disk.
     EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(
                                        0, kTestBlobStorageMaxDiskSpace + 1));
   }
+  {
+    BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+    SetSmallDiskTestMemoryLimits(&controller);
+
+    EXPECT_TRUE(controller.CanReserveQuota(kTestBlobStorageMaxBlobMemorySize));
+    // Since our disk is too small, this should be sent with shared memory.
+    EXPECT_EQ(
+        Strategy::SHARED_MEMORY,
+        controller.DetermineStrategy(0, kTestBlobStorageMaxBlobMemorySize));
+  }
 }
 
 TEST_F(BlobMemoryControllerTest, GrantMemory) {
   const std::string kId = "id";
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
   SetTestMemoryLimits(&controller);
 
   BlobDataBuilder builder(kId);
   builder.AppendFutureData(10);
   builder.AppendFutureData(20);
@@ skipping 102 matching lines @@
   items2.clear();
   EXPECT_EQ(0u, controller.memory_usage());
   items.clear();
   EXPECT_TRUE(file_runner_->HasPendingTask());
   RunFileThreadTasks();
   base::RunLoop().RunUntilIdle();
   EXPECT_EQ(0u, controller.disk_usage());
 }
 
 TEST_F(BlobMemoryControllerTest, NoDiskTooLarge) {
-  const std::string kId = "id";
   BlobMemoryController controller(temp_dir_.GetPath(), nullptr);
   SetTestMemoryLimits(&controller);
 
   EXPECT_FALSE(controller.CanReserveQuota(kTestBlobStorageMaxBlobMemorySize +
                                           kTestBlobStorageMinFileSizeBytes +
                                           1));
 }
 
 TEST_F(BlobMemoryControllerTest, TooLargeForDisk) {
-  const std::string kId = "id";
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
   SetTestMemoryLimits(&controller);
 
   EXPECT_FALSE(controller.CanReserveQuota(kTestBlobStorageMaxDiskSpace + 1));
 }
 
 TEST_F(BlobMemoryControllerTest, CancelMemoryRequest) {
   const std::string kId = "id";
   const std::string kId2 = "id2";
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
@@ skipping 129 matching lines @@
   base::WeakPtr<QuotaAllocationTask> task =
       controller.ReserveFileQuota(items, GetFileCreationCallback());
   EXPECT_TRUE(task);
   EXPECT_EQ(ItemState::QUOTA_REQUESTED, items[0]->state());
   EXPECT_TRUE(file_runner_->HasPendingTask());
   EXPECT_EQ(0u, controller.memory_usage());
   EXPECT_EQ(kBlobSize, controller.disk_usage());
 
   task->Cancel();
   EXPECT_FALSE(task);
-  EXPECT_EQ(kBlobSize, controller.disk_usage());
-  EXPECT_TRUE(file_runner_->HasPendingTask());
+  EXPECT_EQ(0ull, controller.disk_usage());
+
   RunFileThreadTasks();
   base::RunLoop().RunUntilIdle();
-  EXPECT_EQ(0u, controller.disk_usage());
 }
 
 TEST_F(BlobMemoryControllerTest, MultipleFilesPaged) {
   const std::string kId1 = "id";
   const size_t kSize1 = kTestBlobStorageMaxFileSizeBytes;
   char kData1[kSize1];
   std::memset(kData1, 'e', kSize1);
 
   const std::string kId2 = "id2";
   const size_t kSize2 = kTestBlobStorageMaxFileSizeBytes;
@@ skipping 128 matching lines @@
   EXPECT_EQ(
       kTestBlobStorageMaxBlobMemorySize - kTestBlobStorageMinFileSizeBytes,
       controller.memory_usage());
   EXPECT_EQ(
       kTestBlobStorageMaxBlobMemorySize - kTestBlobStorageMinFileSizeBytes,
       controller.disk_usage());
 
   EXPECT_TRUE(memory_quota_result_);
 }
 
+TEST_F(BlobMemoryControllerTest, PagingStopsWhenFull) {
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+  const size_t kTotalBlobStorageSize =
+      kTestBlobStorageMaxDiskSpace + kTestBlobStorageMaxBlobMemorySize;
+
+  const size_t kDataSize = 10u;
+  const size_t kBlobsThatCanFit = kTotalBlobStorageSize / kDataSize;
+  const size_t kNumFastBlobs = kTestBlobStorageMaxBlobMemorySize / kDataSize;
+  char kData[10];
+  memset(kData, 'e', kDataSize);
+
+  // Create all of our blobs.
+  std::vector<scoped_refptr<ShareableBlobDataItem>> all_items;
+  std::vector<base::WeakPtr<QuotaAllocationTask>> memory_tasks;
+  bool memory_requested[kBlobsThatCanFit] = {};
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    BlobDataBuilder builder("fake");
+    builder.AppendData(kData, kDataSize);
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(builder);
+    EXPECT_TRUE(controller.CanReserveQuota(kDataSize));
+    EXPECT_EQ((i < kNumFastBlobs) ? Strategy::NONE_NEEDED : Strategy::IPC,
+              controller.DetermineStrategy(kDataSize, kDataSize))
+        << i;
+    base::WeakPtr<QuotaAllocationTask> memory_task =
+        controller.ReserveMemoryQuota(
+            items, GetMemoryRequestCallbackToOutput(memory_requested + i));
+    if (memory_task) {
+      memory_tasks.push_back(std::move(memory_task));
+    }
+    all_items.insert(all_items.end(), items.begin(), items.end());
+  }
+  // We should have stored all of our memory quota, and no disk yet.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(0ull, controller.disk_usage());
+
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_EQ(i < kBlobsThatCanFit / 3, memory_requested[i]) << i;
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      EXPECT_TRUE(memory_requested[i]);
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+
+  // This will schedule one task. Paging starts as soon as there is enough
+  // memory to page, and multiple pagings can't happen at the same time.
+  EXPECT_EQ(10ull, controller.disk_usage());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // The rest of the tasks should be scheduled.
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // Everything in memory should be on disk, and next batch of memory items
+  // should be granted.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(500ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+
+  // Flag next batch for saving to disk.
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_EQ(i < kBlobsThatCanFit * 2 / 3, memory_requested[i]) << i;
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+
+  // Same as before. One page task is scheduled, so run them twice.
+  EXPECT_EQ(510ull, controller.disk_usage());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // We page one time first, as it blocks paging once it starts.
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+
+  // All quota should be allocated.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(1000ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+
+  // Flag last batch as populated.
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_TRUE(memory_requested[i]);
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+
+  // There should be no more paging to disk, as we've reached the end.
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+
+  // All quota should be allocated still.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(1000ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+}
+
 TEST_F(BlobMemoryControllerTest, DisableDiskWithFileAndMemoryPending) {
   const std::string kFirstMemoryId = "id";
   const uint64_t kFirstMemorySize = kTestBlobStorageMaxBlobMemorySize;
   const std::string kSecondMemoryId = "id2";
   const uint64_t kSecondMemorySize = 1;
   const std::string kFileId = "id2";
   const uint64_t kFileBlobSize = kTestBlobStorageMaxBlobMemorySize;
 
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
   SetTestMemoryLimits(&controller);
@@ skipping 63 matching lines @@
 
   file_items.clear();
   items.clear();
 
   RunFileThreadTasks();
   base::RunLoop().RunUntilIdle();
 
   EXPECT_EQ(0ull, controller.disk_usage());
   EXPECT_EQ(0ull, controller.memory_usage());
 }
+
+TEST_F(BlobMemoryControllerTest, MemoryPressureAndReset) {
+  const std::string kFileId = "id2";
+  const uint64_t kFileBlobSize = kTestBlobStorageMaxBlobMemorySize;
+
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  MockDiskSpaceTestGetter mock_disk_space;
+  controller.set_testing_disk_space(&mock_disk_space);
+
+  BlobDataBuilder file_builder(kFileId);
+  file_builder.AppendFutureFile(0, kFileBlobSize, 0);
+
+  // 1. When we have < kFileBlobSize, then we cancel our request.
+  {
+    SetTestMemoryLimits(&controller);
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(file_builder);
+
+    file_quota_result_ = true;
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(kFileBlobSize - 1));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_FALSE(file_quota_result_);
+    EXPECT_TRUE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(0ull, controller.limits().effective_max_disk);
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+  }
+
+  // 2. When we have < limits.min_available_disk_space(), then we'll modify our
+  //    effective disk space to match our current usage to stop using more disk.
+  {
+    SetTestMemoryLimits(&controller);
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(file_builder);
+
+    file_quota_result_ = false;
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(controller.limits().min_available_disk_space() - 1));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_TRUE(file_quota_result_);
+    EXPECT_TRUE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(kFileBlobSize, controller.limits().effective_max_disk);
+
+    items.clear();
+    files_created_.clear();
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+  }
+
+  // 3. When our desired total disk space is less than we're allowed given the
+  //    minimum disk availability, we shorten the disk space.
+  {
+    SetTestMemoryLimits(&controller);
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(file_builder);
+
+    file_quota_result_ = false;
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(controller.limits().desired_max_disk +
+                         controller.limits().min_available_disk_space() - 1));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_TRUE(file_quota_result_);
+    EXPECT_TRUE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(controller.limits().desired_max_disk - 1,
+              controller.limits().effective_max_disk);
+
+    items.clear();
+    files_created_.clear();
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+  }
+
+  // 4. When we do a file operation after disk has been freed (after we've been
+  //    limited), our effective size grows correctly.
+  {
+    SetTestMemoryLimits(&controller);
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(file_builder);
+
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(controller.limits().min_available_disk_space() - 1));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    // Check the effective limit is constrained.
+    EXPECT_TRUE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(kFileBlobSize, controller.limits().effective_max_disk);
+
+    // Delete the item so we have disk quota.
+    items.clear();
+    files_created_.clear();
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+
+    // Create the same item, but have the disk space report the minimum amount
+    // needed to have the desired disk size.
+    items = CreateSharedDataItems(file_builder);
+
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(kTestBlobStorageMaxDiskSpace +
+                         controller.limits().min_available_disk_space()));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_FALSE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(controller.limits().desired_max_disk,
+              controller.limits().effective_max_disk);
+
+    items.clear();
+    files_created_.clear();
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+  }
+}
 }  // namespace storage