SimpleCache: read small files all at once.

net/disk_cache/entry_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <utility>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/files/file.h"
 #include "base/files/file_util.h"
 #include "base/macros.h"
+#include "base/metrics/field_trial.h"
+#include "base/metrics/field_trial_param_associator.h"
 #include "base/run_loop.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/test/histogram_tester.h"
+#include "base/test/mock_entropy_provider.h"
+#include "base/test/scoped_feature_list.h"
 #include "base/threading/platform_thread.h"
 #include "net/base/completion_callback.h"
 #include "net/base/io_buffer.h"
 #include "net/base/net_errors.h"
 #include "net/base/test_completion_callback.h"
 #include "net/disk_cache/blockfile/backend_impl.h"
 #include "net/disk_cache/blockfile/entry_impl.h"
 #include "net/disk_cache/cache_util.h"
 #include "net/disk_cache/disk_cache_test_base.h"
 #include "net/disk_cache/disk_cache_test_util.h"
@@ skipping 41 matching lines @@
   void DoomNormalEntry();
   void DoomEntryNextToOpenEntry();
   void DoomedEntry(int stream_index);
   void BasicSparseIO();
   void HugeSparseIO();
   void GetAvailableRange();
   void CouldBeSparse();
   void UpdateSparseEntry();
   void DoomSparseEntry();
   void PartialSparseEntry();
-  bool SimpleCacheMakeBadChecksumEntry(const std::string& key, int* data_size);
+  bool SimpleCacheMakeBadChecksumEntry(const std::string& key, int data_size);
   bool SimpleCacheThirdStreamFileExists(const char* key);
   void SyncDoomEntry(const char* key);
 };
 
 // This part of the test runs on the background thread.
 void DiskCacheEntryTest::InternalSyncIOBackground(disk_cache::Entry* entry) {
   const int kSize1 = 10;
   scoped_refptr<net::IOBuffer> buffer1(new net::IOBuffer(kSize1));
   CacheTestFillBuffer(buffer1->data(), kSize1, false);
   EXPECT_EQ(
@@ skipping 2536 matching lines @@
   // it on a doomed entry, if it was previously lazily omitted.
   for (int i = 0; i < disk_cache::kSimpleEntryStreamCount - 1; ++i) {
     EXPECT_THAT(DoomAllEntries(), IsOk());
     DoomedEntry(i);
   }
 }
 
 // Creates an entry with corrupted last byte in stream 0.
 // Requires SimpleCacheMode.
 bool DiskCacheEntryTest::SimpleCacheMakeBadChecksumEntry(const std::string& key,
-                                                         int* data_size) {
+                                                         int data_size) {
   disk_cache::Entry* entry = NULL;
 
   if (CreateEntry(key, &entry) != net::OK || !entry) {
     LOG(ERROR) << "Could not create entry";
     return false;
   }
 
-  const char data[] = "this is very good data";
-  const int kDataSize = arraysize(data);
-  scoped_refptr<net::IOBuffer> buffer(new net::IOBuffer(kDataSize));
-  base::strlcpy(buffer->data(), data, kDataSize);
+  scoped_refptr<net::IOBuffer> buffer(new net::IOBuffer(data_size));
+  memset(buffer->data(), 'A', data_size);
 
-  EXPECT_EQ(kDataSize, WriteData(entry, 1, 0, buffer.get(), kDataSize, false));
+  EXPECT_EQ(data_size, WriteData(entry, 1, 0, buffer.get(), data_size, false));
   entry->Close();
   entry = NULL;
 
   // Corrupt the last byte of the data.
   base::FilePath entry_file0_path = cache_path_.AppendASCII(
       disk_cache::simple_util::GetFilenameFromKeyAndFileIndex(key, 0));
   base::File entry_file0(entry_file0_path,
                          base::File::FLAG_WRITE | base::File::FLAG_OPEN);
   if (!entry_file0.IsValid())
     return false;
 
   int64_t file_offset =
-      sizeof(disk_cache::SimpleFileHeader) + key.size() + kDataSize - 2;
+      sizeof(disk_cache::SimpleFileHeader) + key.size() + data_size - 2;
   EXPECT_EQ(1, entry_file0.Write(file_offset, "X", 1));
-  *data_size = kDataSize;
   return true;
 }
 
-// Tests that the simple cache can detect entries that have bad data.
 TEST_F(DiskCacheEntryTest, SimpleCacheBadChecksum) {
   base::HistogramTester histogram_tester;
   SetSimpleCacheMode();
   InitCache();
 
   const char key[] = "the first key";
-  int size_unused;
-  ASSERT_TRUE(SimpleCacheMakeBadChecksumEntry(key, &size_unused));
+  const int kLargeSize = 50000;
+  ASSERT_TRUE(SimpleCacheMakeBadChecksumEntry(key, kLargeSize));
 
   disk_cache::Entry* entry = NULL;
 
-  // Open the entry.
+  // Open the entry. Can't spot the checksum that quickly with it so
+  // huge.
   ASSERT_THAT(OpenEntry(key, &entry), IsOk());
   ScopedEntryPtr entry_closer(entry);
 
-  const int kReadBufferSize = 200;
-  EXPECT_GE(kReadBufferSize, entry->GetDataSize(1));
-  scoped_refptr<net::IOBuffer> read_buffer(new net::IOBuffer(kReadBufferSize));
+  EXPECT_GE(kLargeSize, entry->GetDataSize(1));
+  scoped_refptr<net::IOBuffer> read_buffer(new net::IOBuffer(kLargeSize));
   EXPECT_EQ(net::ERR_CACHE_CHECKSUM_MISMATCH,
-            ReadData(entry, 1, 0, read_buffer.get(), kReadBufferSize));
+            ReadData(entry, 1, 0, read_buffer.get(), kLargeSize));
   histogram_tester.ExpectUniqueSample(
       "SimpleCache.Http.ReadResult",
       disk_cache::READ_RESULT_SYNC_CHECKSUM_FAILURE, 1);
 }
 
 // Tests that an entry that has had an IO error occur can still be Doomed().
 TEST_F(DiskCacheEntryTest, SimpleCacheErrorThenDoom) {
   base::HistogramTester histogram_tester;
   SetSimpleCacheMode();
   InitCache();
 
   const char key[] = "the first key";
-  int size_unused;
-  ASSERT_TRUE(SimpleCacheMakeBadChecksumEntry(key, &size_unused));
+  const int kLargeSize = 50000;
+  ASSERT_TRUE(SimpleCacheMakeBadChecksumEntry(key, kLargeSize));
 
   disk_cache::Entry* entry = NULL;
 
   // Open the entry, forcing an IO error.
   ASSERT_THAT(OpenEntry(key, &entry), IsOk());
   ScopedEntryPtr entry_closer(entry);
 
-  const int kReadBufferSize = 200;
-  EXPECT_GE(kReadBufferSize, entry->GetDataSize(1));
-  scoped_refptr<net::IOBuffer> read_buffer(new net::IOBuffer(kReadBufferSize));
+  EXPECT_GE(kLargeSize, entry->GetDataSize(1));
+  scoped_refptr<net::IOBuffer> read_buffer(new net::IOBuffer(kLargeSize));
   EXPECT_EQ(net::ERR_CACHE_CHECKSUM_MISMATCH,
-            ReadData(entry, 1, 0, read_buffer.get(), kReadBufferSize));
+            ReadData(entry, 1, 0, read_buffer.get(), kLargeSize));
   histogram_tester.ExpectUniqueSample(
       "SimpleCache.Http.ReadResult",
       disk_cache::READ_RESULT_SYNC_CHECKSUM_FAILURE, 1);
   entry->Doom();  // Should not crash.
 }
 
 bool TruncatePath(const base::FilePath& file_path, int64_t length) {
   base::File file(file_path, base::File::FLAG_WRITE | base::File::FLAG_OPEN);
   if (!file.IsValid())
     return false;
@@ skipping 715 matching lines @@
 
 // Tests that if a read and a following in-flight truncate are both in progress
 // simultaniously that they both can occur successfully. See
 // http://crbug.com/239223
 TEST_F(DiskCacheEntryTest, SimpleCacheInFlightTruncate)  {
   SetSimpleCacheMode();
   InitCache();
 
   const char key[] = "the first key";
 
-  const int kBufferSize = 1024;
+  const int kBufferSize = 50000;  // to avoid quick read

[pasko, 2017/08/04 01:28:34]

The context is not easy to guess from this comment. Perhaps we could make a
constant like kMaxFilePrefetchSize and add a compile_assert to verify that the
buffer size is bigger? A comment can explain that otherwise the
EXPECT_EQ(net::ERR_IO_PENDING, ...) would not work. Or .. maybe the explanation
is different?

[Maks Orlovich, 2017/08/04 18:35:43]

Can't really give an exact number of that (since people could set it as high as
the experiment framework permits...), but I did expand upon the comment.

[pasko, 2017/08/09 12:28:21]

Oh, my comment is about adding a constant and a compile assert only in the test,
and compare it only with the default. This may still break if we set an insanely
large number in fieldtrial_testing_config.json, but I guess we won't do that..
The comment worksforme as well, thanks

   scoped_refptr<net::IOBuffer> write_buffer(new net::IOBuffer(kBufferSize));
   CacheTestFillBuffer(write_buffer->data(), kBufferSize, false);
 
   disk_cache::Entry* entry = NULL;
   ASSERT_THAT(CreateEntry(key, &entry), IsOk());
 
   EXPECT_EQ(kBufferSize,
             WriteData(entry, 1, 0, write_buffer.get(), kBufferSize, false));
   entry->Close();
   entry = NULL;
@@ skipping 110 matching lines @@
 }
 
 // Checking one more scenario of overlapped reading of a bad entry.
 // Differs from the |SimpleCacheMultipleReadersCheckCRC| only by the order of
 // last two reads.
 TEST_F(DiskCacheEntryTest, SimpleCacheMultipleReadersCheckCRC2) {
   SetSimpleCacheMode();
   InitCache();
 
   const char key[] = "key";
-  int size;
-  ASSERT_TRUE(SimpleCacheMakeBadChecksumEntry(key, &size));
+  int size = 50000;
+  ASSERT_TRUE(SimpleCacheMakeBadChecksumEntry(key, size));
 
   scoped_refptr<net::IOBuffer> read_buffer1(new net::IOBuffer(size));
   scoped_refptr<net::IOBuffer> read_buffer2(new net::IOBuffer(size));
 
   // Advance the first reader a little.
   disk_cache::Entry* entry = NULL;
   ASSERT_THAT(OpenEntry(key, &entry), IsOk());
   ScopedEntryPtr entry_closer(entry);
   EXPECT_EQ(1, ReadData(entry, 1, 0, read_buffer1.get(), 1));
 
@@ skipping 724 matching lines @@
   //   CHECK(temp_dir_.CreateUniqueTempDir());
   //   cache_path_ = temp_dir_.GetPath();
   disk_cache::DeleteCache(cache_path_,
                           true /* delete the dir, what we really want*/);
 
   disk_cache::Entry* entry;
   std::string key("a key");
   ASSERT_THAT(CreateEntry(key, &entry), IsOk());
   entry->Close();
 }
+
+class DiskCacheSimplePrefetchTest : public DiskCacheEntryTest {
+ public:
+  DiskCacheSimplePrefetchTest()
+      : field_trial_list_(base::MakeUnique<base::FieldTrialList>(
+            base::MakeUnique<base::MockEntropyProvider>())) {}
+
+  enum { kEntrySize = 1024 };
+
+  void SetUp() override {
+    payload_ = new net::IOBuffer(kEntrySize);
+    CacheTestFillBuffer(payload_->data(), kEntrySize, false);
+    DiskCacheEntryTest::SetUp();
+  }
+
+  void SetupPrefetch(int size) {
+    std::map<std::string, std::string> params;
+    params[disk_cache::kSimplePrefetchBytesParam] = base::IntToString(size);
+    scoped_feature_list_.InitAndEnableFeatureWithParameters(
+        disk_cache::kSimpleCachePrefetchExperiment, params);
+  }
+
+  void InitCacheAndCreateEntry(const std::string& key) {
+    SetSimpleCacheMode();
+    InitCache();
+
+    disk_cache::Entry* entry;
+    ASSERT_EQ(net::OK, CreateEntry(key, &entry));
+    // Use stream 1 since that's what new prefetch stuff is about.
+    ASSERT_EQ(kEntrySize,
+              WriteData(entry, 1, 0, payload_.get(), kEntrySize, false));
+    entry->Close();
+  }
+
+  void TryRead(const std::string& key) {
+    disk_cache::Entry* entry = NULL;
+    ASSERT_THAT(OpenEntry(key, &entry), IsOk());
+    scoped_refptr<net::IOBuffer> read_buf(new net::IOBuffer(kEntrySize));
+    EXPECT_EQ(kEntrySize, ReadData(entry, 1, 0, read_buf.get(), kEntrySize));
+    EXPECT_EQ(0, memcmp(read_buf->data(), payload_->data(), kEntrySize));
+    entry->Close();
+  }
+
+ protected:
+  scoped_refptr<net::IOBuffer> payload_;
+
+  // Need to have the one "global" trial list before we change things.
+  std::unique_ptr<base::FieldTrialList> field_trial_list_;
+  base::test::ScopedFeatureList scoped_feature_list_;
+};
+
+TEST_F(DiskCacheSimplePrefetchTest, NoPrefetch) {
+  base::HistogramTester histogram_tester;
+  SetupPrefetch(0);
+
+  const char kKey[] = "a key";
+  InitCacheAndCreateEntry(kKey);
+  TryRead(kKey);
+
+  histogram_tester.ExpectUniqueSample("SimpleCache.Http.SyncOpenDidPrefetch",
+                                      false, 1);
+  histogram_tester.ExpectUniqueSample(
+      "SimpleCache.Http.ReadStream1FromPrefetched", false, 1);
+}
+
+TEST_F(DiskCacheSimplePrefetchTest, YesPrefetch) {
+  base::HistogramTester histogram_tester;
+  SetupPrefetch(2 * kEntrySize);
+
+  const char kKey[] = "a key";
+  InitCacheAndCreateEntry(kKey);
+  TryRead(kKey);
+
+  histogram_tester.ExpectUniqueSample("SimpleCache.Http.SyncOpenDidPrefetch",
+                                      true, 1);
+  histogram_tester.ExpectUniqueSample(
+      "SimpleCache.Http.ReadStream1FromPrefetched", true, 1);
+}
+
+TEST_F(DiskCacheSimplePrefetchTest, YesPrefetchNoRead) {
+  base::HistogramTester histogram_tester;
+  SetupPrefetch(2 * kEntrySize);
+
+  const char kKey[] = "a key";
+  InitCacheAndCreateEntry(kKey);
+
+  disk_cache::Entry* entry = NULL;
+  ASSERT_THAT(OpenEntry(kKey, &entry), IsOk());
+  entry->Close();
+
+  histogram_tester.ExpectUniqueSample("SimpleCache.Http.SyncOpenDidPrefetch",
+                                      true, 1);
+  // Have to use GetHistogramSamplesSinceCreation here since it's the only
+  // API that handles the cases where the histogram hasn't even been created.
+  std::unique_ptr<base::HistogramSamples> samples(
+      histogram_tester.GetHistogramSamplesSinceCreation(
+          "SimpleCache.Http.ReadStream1FromPrefetched"));
+  EXPECT_EQ(0, samples->TotalCount());
+}
+
+// This makes sure we detect checksum error on entry that's small enough to be
+// prefetched. This is like DiskCacheEntryTest.BadChecksum, but we make sure
+// to configure prefetch explicitly.
+TEST_F(DiskCacheSimplePrefetchTest, BadChecksumSmall) {
+  SetupPrefetch(1024);  // bigger than stuff below.
+  SetSimpleCacheMode();
+  InitCache();
+
+  const char key[] = "the first key";
+  ASSERT_TRUE(SimpleCacheMakeBadChecksumEntry(key, 10));
+
+  disk_cache::Entry* entry = NULL;
+
+  // Open the entry. Since we made a small entry, we will detect the CRC
+  // problem at open.
+  EXPECT_THAT(OpenEntry(key, &entry), IsError(net::ERR_FAILED));
+}