SimpleCache: read small files all at once.

net/disk_cache/simple/simple_synchronous_entry.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/disk_cache/simple/simple_synchronous_entry.h"
 
 #include <algorithm>
 #include <cstring>
 #include <functional>
 #include <limits>
 
 #include "base/compiler_specific.h"
 #include "base/files/file_util.h"
 #include "base/hash.h"
 #include "base/location.h"
+#include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/sha1.h"
 #include "base/timer/elapsed_timer.h"
 #include "crypto/secure_hash.h"
 #include "net/base/hash_value.h"
 #include "net/base/io_buffer.h"
 #include "net/base/net_errors.h"
 #include "net/disk_cache/simple/simple_backend_version.h"
 #include "net/disk_cache/simple/simple_histogram_macros.h"
@@ skipping 236 matching lines @@
     const base::TimeTicks& time_enqueued,
     SimpleEntryCreationResults* out_results) {
   base::TimeTicks start_sync_open_entry = base::TimeTicks::Now();
   SIMPLE_CACHE_UMA(TIMES, "QueueLatency.OpenEntry", cache_type,
                    (start_sync_open_entry - time_enqueued));
 
   SimpleSynchronousEntry* sync_entry =
       new SimpleSynchronousEntry(cache_type, path, key, entry_hash, had_index);
   out_results->result = sync_entry->InitializeForOpen(
       &out_results->entry_stat, &out_results->stream_0_data,
-      &out_results->stream_0_crc32);
+      &out_results->stream_0_crc32, &out_results->stream_1_data,
+      &out_results->stream_1_crc32);
   if (out_results->result != net::OK) {
     sync_entry->Doom();
     delete sync_entry;
     out_results->sync_entry = NULL;
     out_results->stream_0_data = NULL;
     return;
   }
   SIMPLE_CACHE_UMA(TIMES, "DiskOpenLatency", cache_type,
                    base::TimeTicks::Now() - start_sync_open_entry);
   out_results->sync_entry = sync_entry;
@@ skipping 402 matching lines @@
     ++it;
   }
 
   int64_t len_from_start = len - (start - offset);
   *out_start = start;
   *out_result = static_cast<int>(std::min(avail_so_far, len_from_start));
 }
 
 int SimpleSynchronousEntry::CheckEOFRecord(int index,
                                            const SimpleEntryStat& entry_stat,
-                                           uint32_t expected_crc32) const {
+                                           uint32_t expected_crc32) {
   DCHECK(initialized_);
-  uint32_t crc32;
-  bool has_crc32;
-  bool has_key_sha256;
-  int32_t stream_size;
-  int rv = GetEOFRecordData(index, entry_stat, &has_crc32, &has_key_sha256,
-                            &crc32, &stream_size);
+  SimpleFileEOF eof_record;
+  int rv = GetEOFRecordData(nullptr, 0, index, entry_stat, &eof_record);
+
   if (rv != net::OK) {
     Doom();
     return rv;
   }
-  if (has_crc32 && crc32 != expected_crc32) {
+  if ((eof_record.flags & SimpleFileEOF::FLAG_HAS_CRC32) &&
+      eof_record.data_crc32 != expected_crc32) {
     DVLOG(1) << "EOF record had bad crc.";
     RecordCheckEOFResult(cache_type_, CHECK_EOF_RESULT_CRC_MISMATCH);
     Doom();
     return net::ERR_CACHE_CHECKSUM_MISMATCH;
   }
   RecordCheckEOFResult(cache_type_, CHECK_EOF_RESULT_SUCCESS);
   return net::OK;
 }
 
+int SimpleSynchronousEntry::PreReadStreamPayload(
+    char* prefetch_buf,
+    int file_size,
+    int stream_index,
+    int extra_size,
+    const SimpleEntryStat& entry_stat,
+    const SimpleFileEOF& eof_record,
+    scoped_refptr<net::GrowableIOBuffer>* stream_data,
+    uint32_t* out_crc32) {
+  int stream_size = entry_stat.data_size(stream_index);
+  int read_size = stream_size + extra_size;
+  *stream_data = new net::GrowableIOBuffer();
+  (*stream_data)->SetCapacity(read_size);
+  int file_offset = entry_stat.GetOffsetInFile(key_.size(), 0, stream_index);
+  if (!ReadFromFileOrPrefetched(prefetch_buf, file_size, file_offset, read_size,
+                                (*stream_data)->data()))
+    return net::ERR_FAILED;
+
+  // Check the CRC32.
+  uint32_t expected_crc32 =
+      stream_size == 0
+          ? crc32(0, Z_NULL, 0)
+          : crc32(crc32(0, Z_NULL, 0),
+                  reinterpret_cast<const Bytef*>((*stream_data)->data()),
+                  stream_size);
+  if ((eof_record.flags & SimpleFileEOF::FLAG_HAS_CRC32) &&
+      eof_record.data_crc32 != expected_crc32) {
+    DVLOG(1) << "EOF record had bad crc.";
+    RecordCheckEOFResult(cache_type_, CHECK_EOF_RESULT_CRC_MISMATCH);
+    return net::ERR_CACHE_CHECKSUM_MISMATCH;
+  }
+  *out_crc32 = expected_crc32;
+  return net::OK;
+}
+
 void SimpleSynchronousEntry::Close(
     const SimpleEntryStat& entry_stat,
     std::unique_ptr<std::vector<CRCRecord>> crc32s_to_write,
     net::GrowableIOBuffer* stream_0_data) {
   base::ElapsedTimer close_time;
   DCHECK(stream_0_data);
 
   for (std::vector<CRCRecord>::const_iterator it = crc32s_to_write->begin();
        it != crc32s_to_write->end(); ++it) {
     const int stream_index = it->index;
@@ skipping 362 matching lines @@
     }
   }
 
   header_and_key_check_needed_[file_index] = false;
   return true;
 }
 
 int SimpleSynchronousEntry::InitializeForOpen(
     SimpleEntryStat* out_entry_stat,
     scoped_refptr<net::GrowableIOBuffer>* stream_0_data,
-    uint32_t* out_stream_0_crc32) {
+    uint32_t* out_stream_0_crc32,
+    scoped_refptr<net::GrowableIOBuffer>* stream_1_data,
+    uint32_t* out_stream_1_crc32) {
   DCHECK(!initialized_);
   if (!OpenFiles(out_entry_stat)) {
     DLOG(WARNING) << "Could not open platform files for entry.";
     return net::ERR_FAILED;
   }
   for (int i = 0; i < kSimpleEntryFileCount; ++i) {
     if (empty_file_omitted_[i])
       continue;
 
     if (key_.empty()) {
       // If |key_| is empty, we were opened via the iterator interface, without
       // knowing what our key is. We must therefore read the header immediately
       // to discover it, so SimpleEntryImpl can make it available to
       // disk_cache::Entry::GetKey().
       if (!CheckHeaderAndKey(i))
         return net::ERR_FAILED;
     } else {
       // If we do know which key were are looking for, we still need to
       // check that the file actually has it (rather than just being a hash
       // collision or some sort of file system accident), but that can be put
       // off until opportune time: either the read of the footer, or when we
       // start reading in the data, depending on stream # and format revision.
       header_and_key_check_needed_[i] = true;
     }
 
     if (i == 0) {
       // File size for stream 0 has been stored temporarily in data_size[1].
-      int ret_value_stream_0 =
-          ReadAndValidateStream0(out_entry_stat->data_size(1), out_entry_stat,
-                                 stream_0_data, out_stream_0_crc32);
+      int ret_value_stream_0 = ReadAndValidateStream0(
+          out_entry_stat->data_size(1), out_entry_stat, stream_0_data,
+          out_stream_0_crc32, stream_1_data, out_stream_1_crc32);
       if (ret_value_stream_0 != net::OK)
         return ret_value_stream_0;
     } else {
       out_entry_stat->set_data_size(
           2,
           GetDataSizeFromFileSize(key_.size(), out_entry_stat->data_size(2)));
       if (out_entry_stat->data_size(2) < 0) {
         DLOG(WARNING) << "Stream 2 file is too small.";
         return net::ERR_FAILED;
       }
@@ skipping 74 matching lines @@
   }
   RecordSyncCreateResult(CREATE_ENTRY_SUCCESS, had_index_);
   initialized_ = true;
   return net::OK;
 }
 
 int SimpleSynchronousEntry::ReadAndValidateStream0(
     int file_size,
     SimpleEntryStat* out_entry_stat,
     scoped_refptr<net::GrowableIOBuffer>* stream_0_data,
-    uint32_t* out_stream_0_crc32) {
+    uint32_t* out_stream_0_crc32,
+    scoped_refptr<net::GrowableIOBuffer>* stream_1_data,
+    uint32_t* out_stream_1_crc32) {
+  // If the file is sufficiently small, we will prefetch everything --
+  // in which case |prefetch_buf| will be non-null, and we should look at it
+  // rather than call ::Read for the bits.
+  std::unique_ptr<char[]> prefetch_buf;
+  const int kPrefetchLimit = 32 * 1024;
+  if (file_size <= kPrefetchLimit) {
+    prefetch_buf = base::MakeUnique<char[]>(file_size);
+    if (files_[0].Read(0, prefetch_buf.get(), file_size) != file_size)
+      return net::ERR_FAILED;
+  }
+
   // Pretend this file has a null stream zero, and contains the optional key
   // SHA256. This is good enough to read the EOF record on the file, which gives
   // the actual size of stream 0.
   int temp_data_size = GetDataSizeFromFileSize(key_.size(), file_size);
   out_entry_stat->set_data_size(0, 0);
   out_entry_stat->set_data_size(
       1, temp_data_size - sizeof(net::SHA256HashValue) - sizeof(SimpleFileEOF));
 
-  bool has_crc32;
-  bool has_key_sha256;
-  uint32_t read_crc32;
-  int32_t stream_0_size;
-  int ret_value_crc32 =
-      GetEOFRecordData(0, *out_entry_stat, &has_crc32, &has_key_sha256,
-                       &read_crc32, &stream_0_size);
-  if (ret_value_crc32 != net::OK)
-    return ret_value_crc32;
+  SimpleFileEOF stream_0_eof;
+  int rv = GetEOFRecordData(prefetch_buf.get(), file_size, 0, *out_entry_stat,
+                            &stream_0_eof);
+  if (rv != net::OK)
+    return rv;
+
+  bool has_key_sha256 =
+      (stream_0_eof.flags & SimpleFileEOF::FLAG_HAS_KEY_SHA256) ==
+      SimpleFileEOF::FLAG_HAS_KEY_SHA256;
+  int32_t stream_0_size = stream_0_eof.stream_size;
 
   // Calculate and set the real values for the two streams.
   int32_t total_size = out_entry_stat->data_size(1);
   if (!has_key_sha256)
     total_size += sizeof(net::SHA256HashValue);
+
+  // Sanity-check: don't want to be allocating a gigantic buffer > file size
+  // just because of a header field.
   if (stream_0_size > total_size)
     return net::ERR_FAILED;
   out_entry_stat->set_data_size(0, stream_0_size);
   out_entry_stat->set_data_size(1, total_size - stream_0_size);
 
-  // Put stream 0 data in memory.
-  *stream_0_data = new net::GrowableIOBuffer();
-  (*stream_0_data)->SetCapacity(stream_0_size + sizeof(net::SHA256HashValue));
-  int file_offset = out_entry_stat->GetOffsetInFile(key_.size(), 0, 0);
-  int read_size = stream_0_size;
+  // Put stream 0 data in memory --- plus maybe the sha256(key) footer.
+  int extra_stream_0_read = 0;
   if (has_key_sha256)
-    read_size += sizeof(net::SHA256HashValue);
-  if (files_[0].Read(file_offset, (*stream_0_data)->data(), read_size) !=
-      read_size)
-    return net::ERR_FAILED;
+    extra_stream_0_read += sizeof(net::SHA256HashValue);
+  rv = PreReadStreamPayload(prefetch_buf.get(), file_size, 0,
+                            extra_stream_0_read, *out_entry_stat, stream_0_eof,
+                            stream_0_data, out_stream_0_crc32);
+  if (rv != net::OK)
+    return rv;
 
-  // Check the CRC32.
-  uint32_t expected_crc32 =
-      stream_0_size == 0
-          ? crc32(0, Z_NULL, 0)
-          : crc32(crc32(0, Z_NULL, 0),
-                  reinterpret_cast<const Bytef*>((*stream_0_data)->data()),
-                  stream_0_size);
-  if (has_crc32 && read_crc32 != expected_crc32) {
-    DVLOG(1) << "EOF record had bad crc.";
-    RecordCheckEOFResult(cache_type_, CHECK_EOF_RESULT_CRC_MISMATCH);
-    return net::ERR_FAILED;
+  // If prefetch buffer is available, and we have sha256(key) (so we don't need
+  // to look at the header), extract out stream 1 info as well.
+  if (prefetch_buf && has_key_sha256) {
+    SimpleFileEOF stream_1_eof;
+    rv = GetEOFRecordData(prefetch_buf.get(), file_size, 1, *out_entry_stat,
+                          &stream_1_eof);
+    if (rv != net::OK)
+      return rv;
+
+    rv = PreReadStreamPayload(prefetch_buf.get(), file_size, 1, 0,
+                              *out_entry_stat, stream_1_eof, stream_1_data,
+                              out_stream_1_crc32);
+    if (rv != net::OK)
+      return rv;
+  } else {
+    *stream_1_data = nullptr;
   }
-  *out_stream_0_crc32 = expected_crc32;
 
   // If present, check the key SHA256.
   if (has_key_sha256) {
     net::SHA256HashValue hash_value;
     CalculateSHA256OfKey(key_, &hash_value);
     bool matched =
         std::memcmp(&hash_value, (*stream_0_data)->data() + stream_0_size,
                     sizeof(hash_value)) == 0;
     if (!matched) {
       RecordKeySHA256Result(cache_type_, KeySHA256Result::NO_MATCH);
       return net::ERR_FAILED;
     }
     // Elide header check if we verified sha256(key) via footer.
     header_and_key_check_needed_[0] = false;
     RecordKeySHA256Result(cache_type_, KeySHA256Result::MATCHED);
   } else {
     RecordKeySHA256Result(cache_type_, KeySHA256Result::NOT_PRESENT);
   }
 
   // Ensure the key is validated before completion.
   if (!has_key_sha256 && header_and_key_check_needed_[0])
     CheckHeaderAndKey(0);
 
   RecordCheckEOFResult(cache_type_, CHECK_EOF_RESULT_SUCCESS);
   return net::OK;
 }
 
-int SimpleSynchronousEntry::GetEOFRecordData(int index,
+bool SimpleSynchronousEntry::ReadFromFileOrPrefetched(char* prefetch_buf,
+                                                      int file_size,
+                                                      int offset,
+                                                      int size,
+                                                      char* dest) {
+  if (!prefetch_buf) {
+    return files_[0].Read(offset, dest, size) == size;
+  } else {
+    if (offset < 0 || size < 0)
+      return false;
+    base::CheckedNumeric<int> end =
+        base::CheckedNumeric<int>(offset) + size - 1;
+    if (!end.IsValid())
+      return false;
+    if (offset >= file_size || end.ValueOrDie() >= file_size)
+      return false;
+
+    memcpy(dest, prefetch_buf + offset, size);
+    return true;
+  }
+}
+
+int SimpleSynchronousEntry::GetEOFRecordData(char* prefetch_buf,
+                                             int file_size,
+                                             int stream_index,
                                              const SimpleEntryStat& entry_stat,
-                                             bool* out_has_crc32,
-                                             bool* out_has_key_sha256,
-                                             uint32_t* out_crc32,
-                                             int32_t* out_data_size) const {
-  SimpleFileEOF eof_record;
-  int file_offset = entry_stat.GetEOFOffsetInFile(key_.size(), index);
-  int file_index = GetFileIndexFromStreamIndex(index);
-  File* file = const_cast<File*>(&files_[file_index]);
-  if (file->Read(file_offset, reinterpret_cast<char*>(&eof_record),
-                 sizeof(eof_record)) !=
-      sizeof(eof_record)) {
+                                             SimpleFileEOF* eof_record) {
+  int file_offset = entry_stat.GetEOFOffsetInFile(key_.size(), stream_index);
+  int file_index = GetFileIndexFromStreamIndex(stream_index);
+
+  // Should only have prefetch_buf when reading things in file 0.
+  DCHECK(file_index == 0 || !prefetch_buf);
+  bool ok;
+  if (prefetch_buf)
+    ok = ReadFromFileOrPrefetched(prefetch_buf, file_size, file_offset,
+                                  sizeof(SimpleFileEOF),
+                                  reinterpret_cast<char*>(eof_record));
+  else
+    // Need to support files_[1], so can't use ReadFromFileOrPrefetched
+    // in this case.
+    ok = files_[file_index].Read(
+             file_offset, reinterpret_cast<char*>(eof_record),
+             sizeof(SimpleFileEOF)) == sizeof(SimpleFileEOF);
+
+  if (!ok) {
     RecordCheckEOFResult(cache_type_, CHECK_EOF_RESULT_READ_FAILURE);
     return net::ERR_CACHE_CHECKSUM_READ_FAILURE;
   }
 
-  if (eof_record.final_magic_number != kSimpleFinalMagicNumber) {
+  if (eof_record->final_magic_number != kSimpleFinalMagicNumber) {
     RecordCheckEOFResult(cache_type_, CHECK_EOF_RESULT_MAGIC_NUMBER_MISMATCH);
     DVLOG(1) << "EOF record had bad magic number.";
     return net::ERR_CACHE_CHECKSUM_READ_FAILURE;
   }
 
-  if (!base::IsValueInRangeForNumericType<int32_t>(eof_record.stream_size))
+  if (!base::IsValueInRangeForNumericType<int32_t>(eof_record->stream_size))
     return net::ERR_FAILED;
-
-  *out_has_crc32 = (eof_record.flags & SimpleFileEOF::FLAG_HAS_CRC32) ==
-                   SimpleFileEOF::FLAG_HAS_CRC32;
-  *out_has_key_sha256 =
-      (eof_record.flags & SimpleFileEOF::FLAG_HAS_KEY_SHA256) ==
-      SimpleFileEOF::FLAG_HAS_KEY_SHA256;
-  *out_crc32 = eof_record.data_crc32;
-  *out_data_size = eof_record.stream_size;
-  SIMPLE_CACHE_UMA(BOOLEAN, "SyncCheckEOFHasCrc", cache_type_, *out_has_crc32);
+  SIMPLE_CACHE_UMA(BOOLEAN, "SyncCheckEOFHasCrc", cache_type_,
+                   (eof_record->flags & SimpleFileEOF::FLAG_HAS_CRC32) ==
+                       SimpleFileEOF::FLAG_HAS_CRC32);
   return net::OK;
 }
 
 void SimpleSynchronousEntry::Doom() const {
   DeleteFilesForEntryHash(path_, entry_hash_);
 }
 
 // static
 bool SimpleSynchronousEntry::DeleteFileForEntryHash(const FilePath& path,
                                                     const uint64_t entry_hash,
@@ skipping 304 matching lines @@
   range.offset = offset;
   range.length = len;
   range.data_crc32 = data_crc32;
   range.file_offset = data_file_offset;
   sparse_ranges_.insert(std::make_pair(offset, range));
 
   return true;
 }
 
 }  // namespace disk_cache