[BlobStorage] BlobMemoryController & tests

content/browser/blob_storage/blob_memory_controller_unittest.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "storage/browser/blob/blob_memory_controller.h"
+
+#include "base/bind.h"
+#include "base/files/file_util.h"
+#include "base/files/scoped_temp_dir.h"
+#include "base/message_loop/message_loop.h"
+#include "base/run_loop.h"
+#include "base/test/test_simple_task_runner.h"
+#include "base/threading/thread_restrictions.h"
+#include "base/threading/thread_task_runner_handle.h"
+#include "storage/browser/blob/blob_data_builder.h"
+#include "storage/browser/blob/blob_data_item.h"
+#include "storage/browser/blob/shareable_blob_data_item.h"
+#include "storage/common/data_element.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace storage {
+
+using Strategy = BlobMemoryController::Strategy;
+using FileCreationInfo = BlobMemoryController::FileCreationInfo;
+using base::TestSimpleTaskRunner;
+using ItemState = ShareableBlobDataItem::State;
+using QuotaAllocationTask = BlobMemoryController::QuotaAllocationTask;
+
+const std::string kBlobStorageDirectory = "blob_storage";
+const size_t kTestBlobStorageIPCThresholdBytes = 20;
+const size_t kTestBlobStorageMaxSharedMemoryBytes = 50;
+const size_t kTestBlobStorageMaxBlobMemorySize = 500;
+const size_t kTestBlobStorageInFlightMemory = 10;
+const uint64_t kTestBlobStorageMaxDiskSpace = 1000;
+const uint64_t kTestBlobStorageMinFileSizeBytes = 10;
+const uint64_t kTestBlobStorageMaxFileSizeBytes = 100;
+
+class BlobMemoryControllerTest : public testing::Test {
+ protected:
+  BlobMemoryControllerTest() {}
+
+  void SetUp() override {
+    ASSERT_EQ(true, temp_dir_.CreateUniqueTempDir());
+    base::ThreadRestrictions::SetIOAllowed(false);
+  };
+
+  void TearDown() override {
+    files_created_.clear();
+    // Make sure we clean up the files.
+    base::RunLoop().RunUntilIdle();
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+    base::ThreadRestrictions::SetIOAllowed(true);
+    ASSERT_EQ(true, temp_dir_.Delete());
+  }
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> CreateSharedDataItems(
+      const BlobDataBuilder& builder) {
+    std::vector<scoped_refptr<ShareableBlobDataItem>> result;
+    for (size_t i = 0; i < builder.items_.size(); ++i) {
+      result.push_back(make_scoped_refptr(
+          new ShareableBlobDataItem(builder.uuid(), builder.items_[i],
+                                    ShareableBlobDataItem::QUOTA_NEEDED)));
+    }
+    return result;
+  }
+
+  void SetTestMemoryLimits(BlobMemoryController* controller) {
+    BlobStorageLimits limits;
+    limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
+    limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
+    limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
+    limits.in_flight_space = kTestBlobStorageInFlightMemory;
+    limits.max_blob_disk_space = kTestBlobStorageMaxDiskSpace;
+    limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
+    limits.max_file_size = kTestBlobStorageMaxFileSizeBytes;
+    controller->set_limits_for_testing(limits);
+  }
+
+  void SaveFileCreationInfo(bool success, std::vector<FileCreationInfo> info) {
+    file_quota_result_ = success;
+    if (success)
+      files_created_.swap(info);
+  }
+
+  void SaveMemoryRequest(bool success) { memory_quota_result_ = success; }
+
+  BlobMemoryController::FileQuotaRequestCallback GetFileCreationCallback() {
+    return base::Bind(&BlobMemoryControllerTest::SaveFileCreationInfo,
+                      base::Unretained(this));
+  }
+
+  BlobMemoryController::MemoryQuotaRequestCallback GetMemoryRequestCallback() {
+    return base::Bind(&BlobMemoryControllerTest::SaveMemoryRequest,
+                      base::Unretained(this));
+  }
+
+  void CleanUpFileItemsFromController(
+      BlobMemoryController* controller,
+      std::vector<scoped_refptr<ShareableBlobDataItem>> items,
+      const std::string& blob_id) {
+    // Make items free from blob references
+    for (const auto& item : items) {
+      item->referencing_blobs_mutable()->erase(blob_id);
+    }
+
+    // We need the shareable file reference to go away, which schedules tasks
+    // that
+    // eventually calls back to our system and decrements our disk usage.
+    items.clear();
+    base::RunLoop().RunUntilIdle();
+    EXPECT_TRUE(file_runner_->HasPendingTask());
+    RunFileThreadTasks();
+    base::RunLoop().RunUntilIdle();
+  }
+
+  void RunFileThreadTasks() {
+    base::ThreadRestrictions::SetIOAllowed(true);
+    file_runner_->RunPendingTasks();
+    base::ThreadRestrictions::SetIOAllowed(false);
+  }
+
+  bool file_quota_result_ = false;
+  base::ScopedTempDir temp_dir_;
+  std::vector<FileCreationInfo> files_created_;
+  bool memory_quota_result_ = false;
+
+  scoped_refptr<TestSimpleTaskRunner> file_runner_ = new TestSimpleTaskRunner();
+
+  base::MessageLoop fake_io_message_loop_;
+};
+
+TEST_F(BlobMemoryControllerTest, Strategy) {
+  {
+    BlobMemoryController controller(temp_dir_.GetPath(), nullptr);
+    SetTestMemoryLimits(&controller);
+
+    // No transportation needed.
+    EXPECT_EQ(Strategy::NONE_NEEDED, controller.DetermineStrategy(0, 0));
+
+    // IPC.
+    EXPECT_EQ(Strategy::IPC, controller.DetermineStrategy(
+                                 0, kTestBlobStorageIPCThresholdBytes));
+
+    // Shared Memory.
+    EXPECT_EQ(
+        Strategy::SHARED_MEMORY,
+        controller.DetermineStrategy(kTestBlobStorageIPCThresholdBytes,
+                                     kTestBlobStorageMaxSharedMemoryBytes));
+
+    // Not too large, as disk isn't enabled.
+    EXPECT_EQ(
+        Strategy::SHARED_MEMORY,
+        controller.DetermineStrategy(0, kTestBlobStorageMaxBlobMemorySize));
+
+    // Too large.
+    EXPECT_EQ(
+        Strategy::TOO_LARGE,
+        controller.DetermineStrategy(0, kTestBlobStorageMaxBlobMemorySize + 1));
+  }
+  {
+    // Enable disk, and check file strategies.
+    BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+    SetTestMemoryLimits(&controller);
+    EXPECT_EQ(Strategy::FILE, controller.DetermineStrategy(
+                                  0, kTestBlobStorageMaxBlobMemorySize + 1));
+
+    // Too large for disk.
+    EXPECT_EQ(Strategy::TOO_LARGE, controller.DetermineStrategy(
+                                       0, kTestBlobStorageMaxDiskSpace + 1));
+  }
+}
+
+TEST_F(BlobMemoryControllerTest, GrantMemory) {
+  const std::string kId = "id";
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+
+  BlobDataBuilder builder(kId);
+  builder.AppendFutureData(10);
+  builder.AppendFutureData(20);
+  builder.AppendFutureData(30);
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+      CreateSharedDataItems(builder);
+
+  controller.ReserveMemoryQuota(items, GetMemoryRequestCallback());
+  EXPECT_TRUE(memory_quota_result_);
+  EXPECT_EQ(ItemState::QUOTA_GRANTED, items[0]->state());
+  EXPECT_EQ(ItemState::QUOTA_GRANTED, items[1]->state());
+  EXPECT_EQ(ItemState::QUOTA_GRANTED, items[2]->state());
+}
+
+TEST_F(BlobMemoryControllerTest, SimpleMemoryRequest) {
+  const std::string kId = "id";
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+
+  // Add memory item that is the memory quota.
+  BlobDataBuilder builder(kId);
+  builder.AppendFutureData(kTestBlobStorageMaxBlobMemorySize);
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+      CreateSharedDataItems(builder);
+
+  controller.ReserveMemoryQuota(items, GetMemoryRequestCallback());
+  EXPECT_TRUE(memory_quota_result_);
+  memory_quota_result_ = false;
+  EXPECT_EQ(ItemState::QUOTA_GRANTED, items[0]->state());
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+  EXPECT_EQ(kTestBlobStorageMaxBlobMemorySize, controller.memory_usage());
+  EXPECT_EQ(0u, controller.disk_usage());
+
+  items.clear();
+  EXPECT_EQ(0u, controller.memory_usage());
+}
+
+TEST_F(BlobMemoryControllerTest, PageToDisk) {
+  const std::string kId = "id";
+  const std::string kId2 = "id2";
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+
+  char kData[kTestBlobStorageMaxBlobMemorySize];
+  std::memset(kData, kTestBlobStorageMaxBlobMemorySize, 'e');
+
+  // Add memory item that is the memory quota.
+  BlobDataBuilder builder(kId);
+  builder.AppendFutureData(kTestBlobStorageMaxBlobMemorySize);
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+      CreateSharedDataItems(builder);
+
+  controller.ReserveMemoryQuota(items, GetMemoryRequestCallback());
+  EXPECT_TRUE(memory_quota_result_);
+  memory_quota_result_ = false;
+  EXPECT_EQ(ItemState::QUOTA_GRANTED, items[0]->state());
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+  EXPECT_EQ(kTestBlobStorageMaxBlobMemorySize, controller.memory_usage());
+  EXPECT_EQ(0u, controller.disk_usage());
+
+  // Create an item that is just a little too big.
+  BlobDataBuilder builder2(kId2);
+  builder2.AppendFutureData(kTestBlobStorageInFlightMemory + 1);
+
+  // Reserve memory, which should request successfuly but we can't fit it yet
+  // (no callback).
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items2 =
+      CreateSharedDataItems(builder2);
+  controller.ReserveMemoryQuota(items2, GetMemoryRequestCallback());
+  // We don't count the usage yet.
+  EXPECT_EQ(kTestBlobStorageMaxBlobMemorySize, controller.memory_usage());
+  EXPECT_EQ(0u, controller.disk_usage());
+
+  EXPECT_FALSE(memory_quota_result_);
+  EXPECT_EQ(ItemState::QUOTA_REQUESTED, items2[0]->state());
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+
+  // Add our original item as populated so it's paged to disk.
+  items[0]->item()->data_element_ptr()->SetToBytes(
+      kData, kTestBlobStorageMaxBlobMemorySize);
+  items[0]->set_state(ItemState::POPULATED_WITH_QUOTA);
+  controller.NotifyMemoryItemUsed(items[0].get());
+
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  EXPECT_EQ(ItemState::QUOTA_GRANTED, items2[0]->state());
+  EXPECT_EQ(DataElement::TYPE_FILE, items[0]->item()->type());
+  EXPECT_EQ(kTestBlobStorageInFlightMemory + 1, controller.memory_usage());
+  EXPECT_EQ(kTestBlobStorageMaxBlobMemorySize, controller.disk_usage());
+
+  EXPECT_FALSE(controller.CanReserveQuota(kTestBlobStorageMaxDiskSpace));
+
+  items2.clear();
+  EXPECT_EQ(0u, controller.memory_usage());
+  items.clear();
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  EXPECT_EQ(0u, controller.disk_usage());
+}
+
+TEST_F(BlobMemoryControllerTest, NoDiskTooLarge) {
+  const std::string kId = "id";
+  BlobMemoryController controller(temp_dir_.GetPath(), nullptr);
+  SetTestMemoryLimits(&controller);
+
+  EXPECT_FALSE(controller.CanReserveQuota(kTestBlobStorageMaxBlobMemorySize +
+                                          kTestBlobStorageInFlightMemory + 1));
+}
+
+TEST_F(BlobMemoryControllerTest, TooLargeForDisk) {
+  const std::string kId = "id";
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+
+  EXPECT_FALSE(controller.CanReserveQuota(kTestBlobStorageMaxDiskSpace + 1));
+}
+
+TEST_F(BlobMemoryControllerTest, CancelMemoryRequest) {
+  const std::string kId = "id";
+  const std::string kId2 = "id2";
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+
+  char kData[kTestBlobStorageMaxBlobMemorySize];
+  std::memset(kData, kTestBlobStorageMaxBlobMemorySize, 'e');
+
+  // Add memory item that is the memory quota.
+  BlobDataBuilder builder(kId);
+  builder.AppendFutureData(kTestBlobStorageMaxBlobMemorySize);
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+      CreateSharedDataItems(builder);
+
+  controller.ReserveMemoryQuota(items, GetMemoryRequestCallback());
+
+  // Create an item that is just a little too big.
+  BlobDataBuilder builder2(kId2);
+  builder2.AppendFutureData(kTestBlobStorageInFlightMemory + 1);
+
+  // Reserve memory, which should request successfuly but we can't fit it yet
+  // (no callback).
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items2 =
+      CreateSharedDataItems(builder2);
+  base::WeakPtr<QuotaAllocationTask> task =
+      controller.ReserveMemoryQuota(items2, GetMemoryRequestCallback());
+  // We don't count the usage yet.
+  EXPECT_EQ(kTestBlobStorageMaxBlobMemorySize, controller.memory_usage());
+  EXPECT_EQ(0u, controller.disk_usage());
+
+  // Add our original item as populated so we start paging to disk.
+  items[0]->item()->data_element_ptr()->SetToBytes(
+      kData, kTestBlobStorageMaxBlobMemorySize);
+  items[0]->set_state(ItemState::POPULATED_WITH_QUOTA);
+  controller.NotifyMemoryItemUsed(items[0].get());
+
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  EXPECT_TRUE(task);
+
+  task->Cancel();
+  EXPECT_FALSE(task);
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  EXPECT_EQ(ItemState::QUOTA_REQUESTED, items2[0]->state());
+  EXPECT_EQ(DataElement::TYPE_FILE, items[0]->item()->type());
+  EXPECT_EQ(0u, controller.memory_usage());
+  EXPECT_EQ(kTestBlobStorageMaxBlobMemorySize, controller.disk_usage());
+
+  CleanUpFileItemsFromController(&controller, std::move(items), kId);
+  EXPECT_EQ(0u, controller.disk_usage());
+}
+
+TEST_F(BlobMemoryControllerTest, FileRequest) {
+  const std::string kId = "id";
+  const size_t kBlobSize = kTestBlobStorageMaxBlobMemorySize + 1;
+
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+
+  char kData[kBlobSize];
+  std::memset(kData, kBlobSize, 'e');
+
+  // Add item that is the file quota.
+  BlobDataBuilder builder(kId);
+  builder.AppendFutureFile(0, kBlobSize, 0);
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+      CreateSharedDataItems(builder);
+
+  file_quota_result_ = false;
+  base::WeakPtr<QuotaAllocationTask> task =
+      controller.ReserveFileQuota(items, GetFileCreationCallback());
+  EXPECT_TRUE(task);
+  EXPECT_FALSE(file_quota_result_);
+  EXPECT_EQ(ItemState::QUOTA_REQUESTED, items[0]->state());
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  EXPECT_EQ(0u, controller.memory_usage());
+  EXPECT_EQ(kBlobSize, controller.disk_usage());
+
+  EXPECT_FALSE(controller.CanReserveQuota(kTestBlobStorageMaxDiskSpace));
+
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  EXPECT_TRUE(file_quota_result_);
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+  EXPECT_FALSE(task);
+
+  // Do the work to populate the file.
+  EXPECT_EQ(1u, files_created_.size());
+  EXPECT_TRUE(
+      builder.PopulateFutureFile(0, std::move(files_created_[0].file_reference),
+                                 files_created_[0].last_modified));
+  base::ThreadRestrictions::SetIOAllowed(true);
+  files_created_.clear();
+  base::ThreadRestrictions::SetIOAllowed(false);
+  EXPECT_EQ(DataElement::TYPE_FILE, items[0]->item()->type());
+  EXPECT_FALSE(
+      BlobDataBuilder::IsFutureFileItem(items[0]->item()->data_element()));
+
+  builder.Clear();
+  CleanUpFileItemsFromController(&controller, std::move(items), kId);
+  EXPECT_EQ(0u, controller.disk_usage());
+}
+
+TEST_F(BlobMemoryControllerTest, CancelFileRequest) {
+  const std::string kId = "id";
+  const size_t kBlobSize = kTestBlobStorageMaxBlobMemorySize + 1;
+
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+
+  char kData[kBlobSize];
+  std::memset(kData, kBlobSize, 'e');
+
+  // Add memory item that is the memory quota.
+  BlobDataBuilder builder(kId);
+  builder.AppendFutureFile(0, kBlobSize, 0);
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+      CreateSharedDataItems(builder);
+
+  base::WeakPtr<QuotaAllocationTask> task =
+      controller.ReserveFileQuota(items, GetFileCreationCallback());
+  EXPECT_TRUE(task);
+  EXPECT_EQ(ItemState::QUOTA_REQUESTED, items[0]->state());
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  EXPECT_EQ(0u, controller.memory_usage());
+  EXPECT_EQ(kBlobSize, controller.disk_usage());
+
+  task->Cancel();
+  EXPECT_FALSE(task);
+  EXPECT_EQ(kBlobSize, controller.disk_usage());
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+  EXPECT_EQ(0u, controller.disk_usage());
+}
+
+TEST_F(BlobMemoryControllerTest, DisableDiskWithFileAndMemoryPending) {
+  const std::string kFirstMemoryId = "id";
+  const uint64_t kFirstMemorySize = kTestBlobStorageMaxBlobMemorySize;
+  const std::string kSecondMemoryId = "id2";
+  const uint64_t kSecondMemorySize = 1;
+  const std::string kFileId = "id2";
+  const uint64_t kFileBlobSize = kTestBlobStorageMaxBlobMemorySize;
+
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+
+  char kDataMemoryData[kFirstMemorySize];
+  std::memset(kDataMemoryData, kFirstMemorySize, 'e');
+
+  // Add first memory item to fill up some memory quota.
+  BlobDataBuilder builder(kFirstMemoryId);
+  builder.AppendFutureData(kTestBlobStorageMaxBlobMemorySize);
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+      CreateSharedDataItems(builder);
+
+  controller.ReserveMemoryQuota(items, GetMemoryRequestCallback());
+
+  // Create a second memory item that is just a little too big.
+  BlobDataBuilder builder2(kSecondMemoryId);
+  builder2.AppendFutureData(kSecondMemorySize);
+
+  // Reserve memory, which should request successfuly but we can't fit it yet.
+  std::vector<scoped_refptr<ShareableBlobDataItem>> items2 =
+      CreateSharedDataItems(builder2);
+  base::WeakPtr<QuotaAllocationTask> memory_task =
+      controller.ReserveMemoryQuota(items2, GetMemoryRequestCallback());
+  // We don't count the usage yet.
+  EXPECT_EQ(kTestBlobStorageMaxBlobMemorySize, controller.memory_usage());
+  EXPECT_EQ(0u, controller.disk_usage());
+
+  // Add our original item as populated so we start paging it to disk.
+  items[0]->item()->data_element_ptr()->SetToBytes(kDataMemoryData,
+                                                   kFirstMemorySize);
+  items[0]->set_state(ItemState::POPULATED_WITH_QUOTA);
+  controller.NotifyMemoryItemUsed(items[0].get());
+
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  EXPECT_TRUE(memory_task);
+  EXPECT_EQ(kFirstMemorySize, controller.disk_usage());
+
+  // Add our file item now.
+  BlobDataBuilder file_builder(kFileId);
+  file_builder.AppendFutureFile(0, kFileBlobSize, 0);
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> file_items =
+      CreateSharedDataItems(file_builder);
+
+  base::WeakPtr<QuotaAllocationTask> file_task =
+      controller.ReserveFileQuota(file_items, GetFileCreationCallback());
+  EXPECT_TRUE(file_task);
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+
+  // We should have both memory paging tasks and file paging tasks.
+  EXPECT_EQ(kFirstMemorySize, controller.memory_usage());
+  EXPECT_EQ(kFirstMemorySize + kFileBlobSize, controller.disk_usage());
+  file_quota_result_ = true;
+  memory_quota_result_ = true;
+
+  files_created_.clear();
+
+  // Disable paging! This should cancel all file-related tasks and leave us with
+  // only the first memory item.
+  controller.DisableFilePaging(base::File::FILE_ERROR_FAILED);
+  EXPECT_FALSE(file_quota_result_);
+  EXPECT_FALSE(memory_quota_result_);
+  EXPECT_FALSE(file_task);
+  EXPECT_FALSE(memory_task);
+
+  file_items.clear();
+  items.clear();
+
+  RunFileThreadTasks();
+  base::RunLoop().RunUntilIdle();
+
+  EXPECT_EQ(0ull, controller.disk_usage());
+  EXPECT_EQ(0ull, controller.memory_usage());
+}
+}  // namespace storage