[BlobStorage] Enabling disk paging and direct storage.

content/browser/blob_storage/blob_memory_controller_unittest.cc

Index: content/browser/blob_storage/blob_memory_controller_unittest.cc
diff --git a/content/browser/blob_storage/blob_memory_controller_unittest.cc b/content/browser/blob_storage/blob_memory_controller_unittest.cc
index ae0e796c5e34ea4fb555ab52b32889cf6f736503..f755a11e2975d055b0764edcc6a6bc749fa5f373 100644
--- a/content/browser/blob_storage/blob_memory_controller_unittest.cc
+++ b/content/browser/blob_storage/blob_memory_controller_unittest.cc
@@ -16,6 +16,7 @@
 #include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/shareable_blob_data_item.h"
 #include "storage/common/data_element.h"
+#include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace storage {
@@ -25,6 +26,7 @@ using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using base::TestSimpleTaskRunner;
 using ItemState = ShareableBlobDataItem::State;
 using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+using ::testing::Return;
 
 const std::string kBlobStorageDirectory = "blob_storage";
 const size_t kTestBlobStorageIPCThresholdBytes = 20;
@@ -34,6 +36,17 @@ const uint64_t kTestBlobStorageMaxDiskSpace = 1000;
 const uint64_t kTestBlobStorageMinFileSizeBytes = 10;
 const uint64_t kTestBlobStorageMaxFileSizeBytes = 100;
 
+const uint64_t kTestSmallBlobStorageMaxDiskSpace = 100;
+
+class MockDiskSpaceTestGetter
+    : public BlobMemoryController::DiskSpaceTestGetter {
+ public:
+  MockDiskSpaceTestGetter() {}
+  virtual ~MockDiskSpaceTestGetter() {}
+
+  MOCK_CONST_METHOD0(AmountOfFreeDiskSpace, int64_t());
+};
+
 class BlobMemoryControllerTest : public testing::Test {
  protected:
   BlobMemoryControllerTest() {}
@@ -68,7 +81,20 @@ class BlobMemoryControllerTest : public testing::Test {
     limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
     limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
     limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
-    limits.max_blob_disk_space = kTestBlobStorageMaxDiskSpace;
+    limits.desired_max_disk_space = kTestBlobStorageMaxDiskSpace;
+    limits.effective_max_disk_space = kTestBlobStorageMaxDiskSpace;
+    limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
+    limits.max_file_size = kTestBlobStorageMaxFileSizeBytes;
+    controller->set_limits_for_testing(limits);
+  }
+
+  void SetSmallDiskTestMemoryLimits(BlobMemoryController* controller) {
+    BlobStorageLimits limits;
+    limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
+    limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
+    limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
+    limits.desired_max_disk_space = kTestSmallBlobStorageMaxDiskSpace;
+    limits.effective_max_disk_space = kTestSmallBlobStorageMaxDiskSpace;
     limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
     limits.max_file_size = kTestBlobStorageMaxFileSizeBytes;
     controller->set_limits_for_testing(limits);
@@ -81,6 +107,10 @@ class BlobMemoryControllerTest : public testing::Test {
     }
   }
 
+  void SaveMemoryRequestToOutput(bool* output, bool success) {
+    ASSERT_TRUE(output);
+    *output = success;
+  }
   void SaveMemoryRequest(bool success) { memory_quota_result_ = success; }
 
   BlobMemoryController::FileQuotaRequestCallback GetFileCreationCallback() {
@@ -93,6 +123,12 @@ class BlobMemoryControllerTest : public testing::Test {
                       base::Unretained(this));
   }
 
+  BlobMemoryController::MemoryQuotaRequestCallback
+  GetMemoryRequestCallbackToOutput(bool* output) {
+    return base::Bind(&BlobMemoryControllerTest::SaveMemoryRequestToOutput,
+                      base::Unretained(this), output);
+  }
+
   void RunFileThreadTasks() {
     base::ThreadRestrictions::SetIOAllowed(true);
     file_runner_->RunPendingTasks();
@@ -159,6 +195,16 @@ TEST_F(BlobMemoryControllerTest, Strategy) {
     EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(
                                        0, kTestBlobStorageMaxDiskSpace + 1));
   }
+  {
+    BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+    SetSmallDiskTestMemoryLimits(&controller);
+
+    EXPECT_TRUE(controller.CanReserveQuota(kTestBlobStorageMaxBlobMemorySize));
+    // Since our disk is too small, this should be sent with shared memory.
+    EXPECT_EQ(
+        Strategy::SHARED_MEMORY,
+        controller.DetermineStrategy(0, kTestBlobStorageMaxBlobMemorySize));
+  }
 }
 
 TEST_F(BlobMemoryControllerTest, GrantMemory) {
@@ -281,7 +327,6 @@ TEST_F(BlobMemoryControllerTest, PageToDisk) {
 }
 
 TEST_F(BlobMemoryControllerTest, NoDiskTooLarge) {
-  const std::string kId = "id";
   BlobMemoryController controller(temp_dir_.GetPath(), nullptr);
   SetTestMemoryLimits(&controller);
 
@@ -291,7 +336,6 @@ TEST_F(BlobMemoryControllerTest, NoDiskTooLarge) {
 }
 
 TEST_F(BlobMemoryControllerTest, TooLargeForDisk) {
-  const std::string kId = "id";
   BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
   SetTestMemoryLimits(&controller);
 
@@ -441,11 +485,10 @@ TEST_F(BlobMemoryControllerTest, CancelFileRequest) {
 
   task->Cancel();
   EXPECT_FALSE(task);
-  EXPECT_EQ(kBlobSize, controller.disk_usage());
-  EXPECT_TRUE(file_runner_->HasPendingTask());
+  EXPECT_EQ(0ull, controller.disk_usage());
+
   RunFileThreadTasks();
   base::RunLoop().RunUntilIdle();
-  EXPECT_EQ(0u, controller.disk_usage());
 }
 
 TEST_F(BlobMemoryControllerTest, MultipleFilesPaged) {
@@ -594,6 +637,135 @@ TEST_F(BlobMemoryControllerTest, FullEviction) {
   EXPECT_TRUE(memory_quota_result_);
 }
 
+TEST_F(BlobMemoryControllerTest, PagingStopsWhenFull) {
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+  const size_t kTotalBlobStorageSize =
+      kTestBlobStorageMaxDiskSpace + kTestBlobStorageMaxBlobMemorySize;
+
+  const size_t kDataSize = 10u;
+  const size_t kBlobsThatCanFit = kTotalBlobStorageSize / kDataSize;
+  const size_t kNumFastBlobs = kTestBlobStorageMaxBlobMemorySize / kDataSize;
+  char kData[10];
+  memset(kData, 'e', kDataSize);
+
+  // Create all of our blobs.
+  std::vector<scoped_refptr<ShareableBlobDataItem>> all_items;
+  std::vector<base::WeakPtr<QuotaAllocationTask>> memory_tasks;
+  bool memory_requested[kBlobsThatCanFit] = {};
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    BlobDataBuilder builder("fake");
+    builder.AppendData(kData, kDataSize);
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(builder);
+    EXPECT_TRUE(controller.CanReserveQuota(kDataSize));
+    EXPECT_EQ((i < kNumFastBlobs) ? Strategy::NONE_NEEDED : Strategy::IPC,
+              controller.DetermineStrategy(kDataSize, kDataSize))
+        << i;
+    base::WeakPtr<QuotaAllocationTask> memory_task =
+        controller.ReserveMemoryQuota(
+            items, GetMemoryRequestCallbackToOutput(memory_requested + i));
+    if (memory_task) {
+      memory_tasks.push_back(std::move(memory_task));
+    }
+    all_items.insert(all_items.end(), items.begin(), items.end());
+  }
+  // We should have stored all of our memory quota, and no disk yet.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(0ull, controller.disk_usage());
+
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_EQ(i < kBlobsThatCanFit / 3, memory_requested[i]) << i;
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      EXPECT_TRUE(memory_requested[i]);
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+
+  // This will schedule one task. Paging starts as soon as there is enough
+  // memory to page, and multiple pagings can't happen at the same time.
+  EXPECT_EQ(10ull, controller.disk_usage());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // The rest of the tasks should be scheduled.
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // Everything in memory should be on disk, and next batch of memory items
+  // should be granted.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(500ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+
+  // Flag next batch for saving to disk.
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_EQ(i < kBlobsThatCanFit * 2 / 3, memory_requested[i]) << i;
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+
+  // Same as before. One page task is scheduled, so run them twice.
+  EXPECT_EQ(510ull, controller.disk_usage());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  // We page one time first, as it blocks paging once it starts.
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+
+  // All quota should be allocated.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(1000ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+
+  // Flag last batch as populated.
+  for (size_t i = 0; i < kBlobsThatCanFit; i++) {
+    // Note: this can fail if the bot's disk is almost full.
+    EXPECT_TRUE(memory_requested[i]);
+    if (memory_requested[i] &&
+        all_items[i]->state() != ItemState::POPULATED_WITH_QUOTA) {
+      all_items[i]->set_state(ItemState::POPULATED_WITH_QUOTA);
+      std::vector<scoped_refptr<ShareableBlobDataItem>> temp_vector;
+      temp_vector.push_back(all_items[i]);
+      controller.NotifyMemoryItemsUsed(temp_vector);
+    }
+  }
+
+  // There should be no more paging to disk, as we've reached the end.
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+
+  // All quota should be allocated still.
+  EXPECT_EQ(500u, controller.memory_usage());
+  EXPECT_EQ(1000ull, controller.disk_usage());
+
+  // Still can't add anything.
+  EXPECT_FALSE(controller.CanReserveQuota(1u));
+  EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(1u, 1ull));
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+}
+
 TEST_F(BlobMemoryControllerTest, DisableDiskWithFileAndMemoryPending) {
   const std::string kFirstMemoryId = "id";
   const uint64_t kFirstMemorySize = kTestBlobStorageMaxBlobMemorySize;
@@ -677,4 +849,166 @@ TEST_F(BlobMemoryControllerTest, DisableDiskWithFileAndMemoryPending) {
   EXPECT_EQ(0ull, controller.disk_usage());
   EXPECT_EQ(0ull, controller.memory_usage());
 }
+
+TEST_F(BlobMemoryControllerTest, MemoryPressureAndReset) {
+  const std::string kFileId = "id2";
+  const uint64_t kFileBlobSize = kTestBlobStorageMaxBlobMemorySize;
+
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  MockDiskSpaceTestGetter mock_disk_space;
+  controller.set_testing_disk_space(&mock_disk_space);
+
+  BlobDataBuilder file_builder(kFileId);
+  file_builder.AppendFutureFile(0, kFileBlobSize, 0);
+
+  // 1. When we have < kFileBlobSize, then we cancel our request.
+  {
+    SetTestMemoryLimits(&controller);
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(file_builder);
+
+    file_quota_result_ = true;
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())

[Marijn Kruisselbrink, 2016/12/21 17:40:36]

It seems a bit overkill to use gmock for this... You could just have a setter in
your class instead.
But if you are going to use gmock, you probably want to check/clear expectations
(VerifyAndClearExpectations) between the various tests (and possible use
StrictMock as well). That way at least you reduce the risk of getting very
confusing test failures if for example the number of times AmountOfFreeDiskSpace
is called changes.

[dmurph, 2016/12/21 22:27:58]

Done.

+        .WillOnce(Return(kFileBlobSize - 1));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_FALSE(file_quota_result_);
+    EXPECT_TRUE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(0ull, controller.limits().effective_max_disk_space);
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+  }
+
+  // 2. When we have < limits.min_available_disk_space(), then we'll modify our

[Marijn Kruisselbrink, 2016/12/21 17:40:36]

It's not quite clear to me what if any the dependencies between the various
"subtests" here are. It seems like they're mostly independent, in which case it
might make more sense to make them actual separate tests, rather than have them
all in the same test?

[dmurph, 2016/12/21 22:27:58]

Sounds good, done.

+  //    effective disk space to match our current usage to stop using more disk.
+  {
+    SetTestMemoryLimits(&controller);
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(file_builder);
+
+    file_quota_result_ = false;
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(controller.limits().min_available_disk_space() - 1));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_TRUE(file_quota_result_);
+    EXPECT_TRUE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(kFileBlobSize, controller.limits().effective_max_disk_space);
+
+    items.clear();
+    files_created_.clear();
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+  }
+
+  // 3. When our desired total disk space is less than we're allowed given the
+  //    minimum disk availability, we shorten the disk space.
+  {
+    SetTestMemoryLimits(&controller);
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(file_builder);
+
+    file_quota_result_ = false;
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(controller.limits().desired_max_disk_space +
+                         controller.limits().min_available_disk_space() - 1));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_TRUE(file_quota_result_);
+    EXPECT_TRUE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(controller.limits().desired_max_disk_space - 1,
+              controller.limits().effective_max_disk_space);
+
+    items.clear();
+    files_created_.clear();
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+  }
+
+  // 4. When we do a file operation after disk has been freed (after we've been
+  //    limited), our effective size grows correctly.
+  {
+    SetTestMemoryLimits(&controller);
+
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(file_builder);
+
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(controller.limits().min_available_disk_space() - 1));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    // Check the effective limit is constrained.
+    EXPECT_TRUE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(kFileBlobSize, controller.limits().effective_max_disk_space);
+
+    // Delete the item so we have disk quota.
+    items.clear();
+    files_created_.clear();
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+
+    // Create the same item, but have the disk space report the minimum amount
+    // needed to have the desired disk size.
+    items = CreateSharedDataItems(file_builder);
+
+    controller.ReserveFileQuota(items, GetFileCreationCallback());
+
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    EXPECT_CALL(mock_disk_space, AmountOfFreeDiskSpace())
+        .WillOnce(Return(kTestBlobStorageMaxDiskSpace +
+                         controller.limits().min_available_disk_space()));
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_FALSE(controller.limits().IsDiskSpaceConstrained());
+    EXPECT_EQ(controller.limits().desired_max_disk_space,
+              controller.limits().effective_max_disk_space);
+
+    items.clear();
+    files_created_.clear();
+
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+
+    EXPECT_EQ(0ull, controller.disk_usage());
+    EXPECT_EQ(0ull, controller.memory_usage());
+  }
+}
 }  // namespace storage