SimpleCache: read small files all at once.

net/disk_cache/simple/simple_synchronous_entry.h

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef NET_DISK_CACHE_SIMPLE_SIMPLE_SYNCHRONOUS_ENTRY_H_
 #define NET_DISK_CACHE_SIMPLE_SIMPLE_SYNCHRONOUS_ENTRY_H_
 
 #include <stdint.h>
 
 #include <algorithm>
 #include <map>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
+#include "base/feature_list.h"
 #include "base/files/file.h"
 #include "base/files/file_path.h"
 #include "base/gtest_prod_util.h"
 #include "base/memory/ref_counted.h"
+#include "base/strings/string_piece_forward.h"
 #include "base/time/time.h"
 #include "net/base/cache_type.h"
 #include "net/base/net_export.h"
 #include "net/disk_cache/simple/simple_entry_format.h"
 
 namespace net {
 class GrowableIOBuffer;
 class IOBuffer;
 }
 
 FORWARD_DECLARE_TEST(DiskCacheBackendTest, SimpleCacheEnumerationLongKeys);
 
 namespace disk_cache {
 
+NET_EXPORT_PRIVATE extern const base::Feature kSimpleCachePrefetchExperiment;
+NET_EXPORT_PRIVATE extern const char kSimplePrefetchBytesParam[];
+
+// Returns how large a file would get prefetched on reading the entry.
+// If the experiment is disabled, returns 0.
+NET_EXPORT_PRIVATE int GetSimpleCachePrefetchSize();
+
 class SimpleSynchronousEntry;
 
 // This class handles the passing of data about the entry between
 // SimpleEntryImplementation and SimpleSynchronousEntry and the computation of
 // file offsets based on the data size for all streams.
 class NET_EXPORT_PRIVATE SimpleEntryStat {
  public:
   SimpleEntryStat(base::Time last_used,
                   base::Time last_modified,
                   const int32_t data_size[],
@@ skipping 21 matching lines @@
     sparse_data_size_ = sparse_data_size;
   }
 
  private:
   base::Time last_used_;
   base::Time last_modified_;
   int32_t data_size_[kSimpleEntryStreamCount];
   int32_t sparse_data_size_;
 };
 
+struct SimpleStreamPrefetchData {

[pasko, 2017/08/04 01:28:35]

Arguably we should use "a struct only for passive objects that carry data;
everything else is a class", and scoped_refptr is part of it and is _not_ a
passive object. Magical things happen inside the object pointed to by |data|
when we copy the struct.

class?

[Maks Orlovich, 2017/08/07 13:09:01]

Is it that different from having an std::string member?

[pasko, 2017/08/09 12:28:21]

I think std::string and scoped_refptr equally make me want to make it a class.
Also there is " If in doubt, make it a class" in the relevant part of the
styleguide...

+  SimpleStreamPrefetchData();
+  ~SimpleStreamPrefetchData();
+
+  scoped_refptr<net::GrowableIOBuffer> data;
+  uint32_t stream_crc32;
+};
+
 struct SimpleEntryCreationResults {

[pasko, 2017/08/04 01:28:34]

then this should also be a class?

   explicit SimpleEntryCreationResults(SimpleEntryStat entry_stat);
   ~SimpleEntryCreationResults();
 
   SimpleSynchronousEntry* sync_entry;
-  scoped_refptr<net::GrowableIOBuffer> stream_0_data;
+
+  // Expectation is that [0] will always be filled in, but [1] might not be.
+  SimpleStreamPrefetchData stream_prefetch_data[2];
+
   SimpleEntryStat entry_stat;
-  uint32_t stream_0_crc32;
   int result;
 };
 
 // Worker thread interface to the very simple cache. This interface is not
 // thread safe, and callers must ensure that it is only ever accessed from
 // a single thread between synchronization points.
 class SimpleSynchronousEntry {
  public:
   struct CRCRecord {
     CRCRecord();
@@ skipping 87 matching lines @@
   // requested.
   void ReadData(const EntryOperationData& in_entry_op,
                 CRCRequest* crc_request,
                 SimpleEntryStat* entry_stat,
                 net::IOBuffer* out_buf,
                 int* out_result);
   void WriteData(const EntryOperationData& in_entry_op,
                  net::IOBuffer* in_buf,
                  SimpleEntryStat* out_entry_stat,
                  int* out_result);
-  int CheckEOFRecord(int index,
+  int CheckEOFRecord(int stream_index,
                      const SimpleEntryStat& entry_stat,
-                     uint32_t expected_crc32) const;
+                     uint32_t expected_crc32);
 
   void ReadSparseData(const EntryOperationData& in_entry_op,
                       net::IOBuffer* out_buf,
                       base::Time* out_last_used,
                       int* out_result);
   void WriteSparseData(const EntryOperationData& in_entry_op,
                        net::IOBuffer* in_buf,
                        uint64_t max_sparse_data_size,
                        SimpleEntryStat* out_entry_stat,
                        int* out_result);
@@ skipping 70 matching lines @@
   void CloseFiles();
 
   // Read the header and key at the beginning of the file, and validate that
   // they are correct. If this entry was opened with a key, the key is checked
   // for a match. If not, then the |key_| member is set based on the value in
   // this header. Records histograms if any check is failed.
   bool CheckHeaderAndKey(int file_index);
 
   // Returns a net error, i.e. net::OK on success.
   int InitializeForOpen(SimpleEntryStat* out_entry_stat,
-                        scoped_refptr<net::GrowableIOBuffer>* stream_0_data,
-                        uint32_t* out_stream_0_crc32);
+                        SimpleStreamPrefetchData* stream_prefetch_data);
 
   // Writes the header and key to a newly-created stream file. |index| is the
   // index of the stream. Returns true on success; returns false and sets
   // |*out_result| on failure.
   bool InitializeCreatedFile(int index, CreateEntryResult* out_result);
 
   // Returns a net error, including net::OK on success and net::FILE_EXISTS
   // when the entry already exists.
   int InitializeForCreate(SimpleEntryStat* out_entry_stat);
 
   // Allocates and fills a buffer with stream 0 data in |stream_0_data|, then
-  // checks its crc32.
-  int ReadAndValidateStream0(
+  // checks its crc32. May also optionally read in |stream_1_data| and its
+  // crc, but might decide not to.
+  int ReadAndValidateStream0AndMaybe1(
       int file_size,
       SimpleEntryStat* out_entry_stat,
-      scoped_refptr<net::GrowableIOBuffer>* stream_0_data,
-      uint32_t* out_stream_0_crc32);
+      SimpleStreamPrefetchData* stream_prefetch_data);

[pasko, 2017/08/04 01:28:35]

We should explain that |stream_prefetch_data| is a two-element array, otherwise
the declaration is confusing. Also, how about the array syntax:

SimpleStreamPrefetchData stream_prefetch_data[2]

[Maks Orlovich, 2017/08/04 18:35:44]

Done (and elsewhere)

 
-  int GetEOFRecordData(int index,
+  // Reads the EOF record for stream |stream_index| from the appropriate file,
+  // or in memory prefetched version in |file_0_prefetch|
+  // using |entry_stat| for layout info, and putting result into
+  // |*eof_record|. Sanity-checks the result.
+  // Returns net status, and records any failures to UMA.
+  int GetEOFRecordData(base::StringPiece file_0_prefetch,
+                       int stream_index,
                        const SimpleEntryStat& entry_stat,
-                       bool* out_has_crc32,
-                       bool* out_has_key_sha256,
-                       uint32_t* out_crc32,
-                       int32_t* out_data_size) const;
+                       SimpleFileEOF* eof_record);
+
+  // Reads either from |file_0_prefetch| or files_[file_index].
+  // Range-checks all the in-memory reads.
+  bool ReadFromFileOrPrefetched(base::StringPiece file_0_prefetch,
+                                int file_index,
+                                int offset,
+                                int size,
+                                char* dest);
+
+  // Extracts out the payload of stream |stream_index|, reading either from
+  // |file_0_prefetch|, if available, or the file. |entry_stat| will be used to
+  // determine file layout, though |extra_size| additional bytes will be read
+  // past the stream payload end.
+  //
+  // |*stream_data| will be pointed to a fresh buffer with the results,
+  // and |*out_crc32| will get the checksum, which will be verified against
+  // |eof_record|.
+  int PreReadStreamPayload(base::StringPiece file_0_prefetch,
+                           int stream_index,
+                           int extra_size,
+                           const SimpleEntryStat& entry_stat,
+                           const SimpleFileEOF& eof_record,
+                           scoped_refptr<net::GrowableIOBuffer>* stream_data,

[pasko, 2017/08/04 01:28:34]

why not producing SimpleStreamPrefetchData instead of the last two output
params?

[Maks Orlovich, 2017/08/04 18:35:44]

Done.

+                           uint32_t* out_crc32);
+
   void Doom() const;
 
   // Opens the sparse data file and scans it if it exists.
   bool OpenSparseFileIfExists(int32_t* out_sparse_data_size);
 
   // Creates and initializes the sparse data file.
   bool CreateSparseFile();
 
   // Closes the sparse data file.
   void CloseSparseFile();
@@ skipping 69 matching lines @@
   int64_t sparse_tail_offset_;
 
   // True if the entry was created, or false if it was opened. Used to log
   // SimpleCache.*.EntryCreatedWithStream2Omitted only for created entries.
   bool files_created_;
 };
 
 }  // namespace disk_cache
 
 #endif  // NET_DISK_CACHE_SIMPLE_SIMPLE_SYNCHRONOUS_ENTRY_H_