[CacheStorage] Pad and bin opaque resource sizes.

content/browser/cache_storage/cache_storage_cache.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/cache_storage/cache_storage_cache.h"
 
 #include <stddef.h>
 #include <algorithm>
+#include <functional>
 #include <limits>
 #include <memory>
 #include <string>
 #include <utility>
 
 #include "base/barrier_closure.h"
 #include "base/bind_helpers.h"
 #include "base/files/file_path.h"
 #include "base/guid.h"
 #include "base/macros.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "content/browser/cache_storage/cache_storage.pb.h"
 #include "content/browser/cache_storage/cache_storage_blob_to_disk_cache.h"
 #include "content/browser/cache_storage/cache_storage_cache_handle.h"
 #include "content/browser/cache_storage/cache_storage_cache_observer.h"
 #include "content/browser/cache_storage/cache_storage_scheduler.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/common/referrer.h"
+#include "crypto/hmac.h"
+#include "crypto/symmetric_key.h"
 #include "net/base/completion_callback.h"
 #include "net/base/io_buffer.h"
 #include "net/base/net_errors.h"
 #include "net/disk_cache/disk_cache.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "storage/browser/blob/blob_data_builder.h"
 #include "storage/browser/blob/blob_data_handle.h"
 #include "storage/browser/blob/blob_storage_context.h"
 #include "storage/browser/blob/blob_url_request_job_factory.h"
 #include "storage/browser/quota/quota_manager_proxy.h"
@@ skipping 207 matching lines @@
       metadata.response().status_text(),
       ProtoResponseTypeToWebResponseType(metadata.response().response_type()),
       std::move(headers), "", 0, blink::kWebServiceWorkerResponseErrorUnknown,
       base::Time::FromInternalValue(metadata.response().response_time()),
       true /* is_in_cache_storage */, cache_name,
       base::MakeUnique<ServiceWorkerHeaderList>(
           metadata.response().cors_exposed_header_names().begin(),
           metadata.response().cors_exposed_header_names().end()));
 }
 
+// The size of opaque (non-cors) resource responses are padded in order
+// to obfuscate their actual size.
+bool ShouldPadResponseType(blink::WebServiceWorkerResponseType response_type,
+                           bool has_urls) {
+  switch (response_type) {
+    case blink::kWebServiceWorkerResponseTypeBasic:
+    case blink::kWebServiceWorkerResponseTypeCORS:
+    case blink::kWebServiceWorkerResponseTypeDefault:
+    case blink::kWebServiceWorkerResponseTypeError:
+      return false;
+    case blink::kWebServiceWorkerResponseTypeOpaque:
+    case blink::kWebServiceWorkerResponseTypeOpaqueRedirect:
+      return has_urls;
+  }
+  NOTREACHED();
+  return false;
+}
+
+bool ShouldPadResourceSize(const content::proto::CacheResponse* response) {
+  return ShouldPadResponseType(
+      ProtoResponseTypeToWebResponseType(response->response_type()),
+      response->url_list_size());
+}
+
+bool ShouldPadResourceSize(const ServiceWorkerResponse* response) {
+  return ShouldPadResponseType(response->response_type,
+                               !response->url_list.empty());
+}
+
+//

[jkarlin, 2017/07/21 21:18:08]

Remove this line

[cmumford, 2017/07/27 23:33:24]

Done.

+// If the way that a cache's padding is calculated increment this version.
+//
+// History:
+//
+//   1: Uniform random 400K.
+//

[jkarlin, 2017/07/21 21:18:09]

Remove this line

[cmumford, 2017/07/27 23:33:24]

Done.

+const int32_t kCachePaddingAlgorithmVersion = 1;
+
+template <typename T>
+int64_t CalculateResponsePaddingInternal(
+    const T* response,

[jkarlin, 2017/07/21 21:18:08]

Hmm, response is only passed in to be DCHECK'd? That doesn't seem right. Perhaps
run the DCHECK at the calling site? 

[cmumford, 2017/07/27 23:33:24]

Done.

+    const std::string& response_url,
+    const crypto::SymmetricKey* padding_key,
+    int side_data_size) {
+  const uint64_t kPaddingRange = 400 * 1024;  // Increment version if changed.
+
+  DCHECK(ShouldPadResourceSize(response));
+  DCHECK(!response_url.empty());
+
+  std::string data = side_data_size ? response_url + "METADATA" : response_url;

[jkarlin, 2017/07/21 21:18:08]

URLs can be long, let's avoid this string copy in the cases where we can.

[cmumford, 2017/07/27 23:33:24]

Done.

+  crypto::HMAC hmac(crypto::HMAC::SHA256);
+  std::vector<uint8_t> digest(hmac.DigestLength());
+  if (!hmac.Init(padding_key) || !hmac.Sign(data, &digest[0], digest.size())) {
+    LOG(FATAL) << "Failed to calculate HMAC.";
+  }
+  uint64_t val = *(reinterpret_cast<uint64_t*>(&digest[0]));

[jkarlin, 2017/07/21 21:18:08]

Let's DCHECK that digest is at least uint64_t in size for peace of mind

[cmumford, 2017/07/27 23:33:24]

Done.

+  return val % kPaddingRange;
+}
+
+int64_t CalculateResponsePaddingInternal(
+    const ::content::proto::CacheResponse* response,
+    const crypto::SymmetricKey* padding_key,
+    int side_data_size) {
+  DCHECK(ShouldPadResourceSize(response));
+  const std::string& url = response->url_list(response->url_list_size() - 1);
+  return CalculateResponsePaddingInternal(response, url, padding_key,
+                                          side_data_size);
+}
+
 }  // namespace
 
 // The state needed to pass between CacheStorageCache::Put callbacks.
 struct CacheStorageCache::PutContext {
   PutContext(std::unique_ptr<ServiceWorkerFetchRequest> request,
              std::unique_ptr<ServiceWorkerResponse> response,
              std::unique_ptr<storage::BlobDataHandle> blob_data_handle,
              CacheStorageCache::ErrorCallback callback)
       : request(std::move(request)),
         response(std::move(response)),
@@ skipping 17 matching lines @@
   std::unique_ptr<ServiceWorkerFetchRequest> request;
   std::unique_ptr<ServiceWorkerResponse> response;
   std::unique_ptr<storage::BlobDataHandle> blob_handle;
   disk_cache::ScopedEntryPtr entry;
   base::Time entry_time;
 };
 
 struct CacheStorageCache::QueryCacheContext {
   QueryCacheContext(std::unique_ptr<ServiceWorkerFetchRequest> request,
                     const CacheStorageCacheQueryParams& options,
-                    QueryCacheCallback callback)
+                    QueryCacheCallback callback,
+                    QueryTypes query_types)
       : request(std::move(request)),
         options(options),
         callback(std::move(callback)),
+        query_types(query_types),
         matches(base::MakeUnique<QueryCacheResults>()) {}
 
   ~QueryCacheContext() {
     // If the CacheStorageCache is deleted before a backend operation to open
     // an entry completes, the callback won't be run and the resulting entry
     // will be leaked unless we close it here.
     if (enumerated_entry) {
       enumerated_entry->Close();
       enumerated_entry = nullptr;
     }
   }
 
   // Input to QueryCache
   std::unique_ptr<ServiceWorkerFetchRequest> request;
   CacheStorageCacheQueryParams options;
   QueryCacheCallback callback;
-  QueryCacheType query_type;
+  QueryTypes query_types = 0x0;

[jkarlin, 2017/07/21 21:18:08]

0?

[cmumford, 2017/07/27 23:33:24]

Done.

   size_t estimated_out_bytes = 0;
 
   // Iteration state
   std::unique_ptr<disk_cache::Backend::Iterator> backend_iterator;
   disk_cache::Entry* enumerated_entry = nullptr;
 
   // Output of QueryCache
   std::unique_ptr<std::vector<QueryCacheResult>> matches;
 
  private:
   DISALLOW_COPY_AND_ASSIGN(QueryCacheContext);
 };
 
 // static
 std::unique_ptr<CacheStorageCache> CacheStorageCache::CreateMemoryCache(
     const GURL& origin,
     const std::string& cache_name,
     CacheStorage* cache_storage,
     scoped_refptr<net::URLRequestContextGetter> request_context_getter,
     scoped_refptr<storage::QuotaManagerProxy> quota_manager_proxy,
-    base::WeakPtr<storage::BlobStorageContext> blob_context) {
+    base::WeakPtr<storage::BlobStorageContext> blob_context,
+    std::unique_ptr<crypto::SymmetricKey> cache_padding_key) {
   CacheStorageCache* cache = new CacheStorageCache(
       origin, cache_name, base::FilePath(), cache_storage,
       std::move(request_context_getter), std::move(quota_manager_proxy),
-      blob_context, 0 /* cache_size */);
+      blob_context, 0 /* cache_size */, 0 /* cache_padding */,
+      std::move(cache_padding_key));
   cache->SetObserver(cache_storage);
   cache->InitBackend();
   return base::WrapUnique(cache);
 }
 
 // static
 std::unique_ptr<CacheStorageCache> CacheStorageCache::CreatePersistentCache(
     const GURL& origin,
     const std::string& cache_name,
     CacheStorage* cache_storage,
     const base::FilePath& path,
     scoped_refptr<net::URLRequestContextGetter> request_context_getter,
     scoped_refptr<storage::QuotaManagerProxy> quota_manager_proxy,
     base::WeakPtr<storage::BlobStorageContext> blob_context,
-    int64_t cache_size) {
+    int64_t cache_size,
+    int64_t cache_padding,
+    std::unique_ptr<crypto::SymmetricKey> cache_padding_key) {
   CacheStorageCache* cache = new CacheStorageCache(
       origin, cache_name, path, cache_storage,
       std::move(request_context_getter), std::move(quota_manager_proxy),
-      blob_context, cache_size);
+      blob_context, cache_size, cache_padding, std::move(cache_padding_key));
   cache->SetObserver(cache_storage), cache->InitBackend();
   return base::WrapUnique(cache);
 }
 
 base::WeakPtr<CacheStorageCache> CacheStorageCache::AsWeakPtr() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
 void CacheStorageCache::Match(
     std::unique_ptr<ServiceWorkerFetchRequest> request,
@@ skipping 225 matching lines @@
 }
 
 CacheStorageCache::CacheStorageCache(
     const GURL& origin,
     const std::string& cache_name,
     const base::FilePath& path,
     CacheStorage* cache_storage,
     scoped_refptr<net::URLRequestContextGetter> request_context_getter,
     scoped_refptr<storage::QuotaManagerProxy> quota_manager_proxy,
     base::WeakPtr<storage::BlobStorageContext> blob_context,
-    int64_t cache_size)
+    int64_t cache_size,
+    int64_t cache_padding,
+    std::unique_ptr<crypto::SymmetricKey> cache_padding_key)
     : origin_(origin),
       cache_name_(cache_name),
       path_(path),
       cache_storage_(cache_storage),
       request_context_getter_(std::move(request_context_getter)),
       quota_manager_proxy_(std::move(quota_manager_proxy)),
       blob_storage_context_(blob_context),
       scheduler_(
           new CacheStorageScheduler(CacheStorageSchedulerClient::CLIENT_CACHE)),
       cache_size_(cache_size),
+      cache_padding_(cache_padding),
+      cache_padding_key_(std::move(cache_padding_key)),
       max_query_size_bytes_(kMaxQueryCacheResultBytes),
       cache_observer_(nullptr),
       memory_only_(path.empty()),
       weak_ptr_factory_(this) {
   DCHECK(!origin_.is_empty());
   DCHECK(quota_manager_proxy_.get());
+  DCHECK(cache_padding_key_.get());
 
   quota_manager_proxy_->NotifyOriginInUse(origin_);
 }
 
 void CacheStorageCache::QueryCache(
     std::unique_ptr<ServiceWorkerFetchRequest> request,
     const CacheStorageCacheQueryParams& options,
-    QueryCacheType query_type,
+    QueryTypes query_types,
     QueryCacheCallback callback) {
-  DCHECK_NE(BACKEND_UNINITIALIZED, backend_state_);
-  if (backend_state_ != BACKEND_OPEN) {
+  DCHECK_NE(
+      QUERY_CACHE_ENTRIES | QUERY_CACHE_RESPONSES_WITH_BODIES,
+      query_types & (QUERY_CACHE_ENTRIES | QUERY_CACHE_RESPONSES_WITH_BODIES));
+  if (backend_state_ == BACKEND_CLOSED) {
     std::move(callback).Run(CACHE_STORAGE_ERROR_STORAGE,
                             std::unique_ptr<QueryCacheResults>());
     return;
   }
 
   if (!options.ignore_method && request && !request->method.empty() &&
       request->method != "GET") {
     std::move(callback).Run(CACHE_STORAGE_OK,
                             base::MakeUnique<QueryCacheResults>());
     return;
   }
 
   ServiceWorkerFetchRequest* request_ptr = request.get();
-  std::unique_ptr<QueryCacheContext> query_cache_context(
-      new QueryCacheContext(std::move(request), options, std::move(callback)));
-  query_cache_context->query_type = query_type;
+  std::unique_ptr<QueryCacheContext> query_cache_context(new QueryCacheContext(
+      std::move(request), options, std::move(callback), query_types));
 
   if (query_cache_context->request &&
       !query_cache_context->request->url.is_empty() && !options.ignore_search) {
     // There is no need to scan the entire backend, just open the exact
     // URL.
     disk_cache::Entry** entry_ptr = &query_cache_context->enumerated_entry;
     net::CompletionCallback open_entry_callback =
         base::AdaptCallbackForRepeating(
             base::BindOnce(&CacheStorageCache::QueryCacheDidOpenFastPath,
                            weak_ptr_factory_.GetWeakPtr(),
@@ skipping 62 matching lines @@
   if (rv < 0) {
     std::move(query_cache_context->callback)
         .Run(CACHE_STORAGE_ERROR_STORAGE,
              std::move(query_cache_context->matches));
     return;
   }
 
   disk_cache::ScopedEntryPtr entry(query_cache_context->enumerated_entry);
   query_cache_context->enumerated_entry = nullptr;
 
-  if (backend_state_ != BACKEND_OPEN) {
+  if (backend_state_ == BACKEND_CLOSED) {
     std::move(query_cache_context->callback)
         .Run(CACHE_STORAGE_ERROR_NOT_FOUND,
              std::move(query_cache_context->matches));
     return;
   }
 
   if (query_cache_context->request &&
       !query_cache_context->request->url.is_empty()) {
     GURL requestURL = query_cache_context->request->url;
     GURL cachedURL = GURL(entry->GetKey());
@@ skipping 41 matching lines @@
 
   if (query_cache_context->request &&
       !query_cache_context->options.ignore_vary &&
       !VaryMatches(query_cache_context->request->headers,
                    match->request->headers, match->response->headers)) {
     query_cache_context->matches->pop_back();
     QueryCacheOpenNextEntry(std::move(query_cache_context));
     return;
   }
 
-  if (query_cache_context->query_type == QueryCacheType::CACHE_ENTRIES) {
+  if (query_cache_context->query_types & QUERY_CACHE_ENTRIES)
+    match->entry = std::move(entry);
+
+  if (query_cache_context->query_types & QUERY_CACHE_REQUESTS) {
+    query_cache_context->estimated_out_bytes +=
+        match->request->EstimatedStructSize();
+    if (query_cache_context->estimated_out_bytes > max_query_size_bytes_) {
+      std::move(query_cache_context->callback)
+          .Run(CACHE_STORAGE_ERROR_QUERY_TOO_LARGE,
+               std::unique_ptr<QueryCacheResults>());
+      return;
+    }
+  } else {
     match->request.reset();
-    match->response.reset();
-    match->entry = std::move(entry);
-    QueryCacheOpenNextEntry(std::move(query_cache_context));
-    return;
   }
 
-  query_cache_context->estimated_out_bytes +=
-      match->request->EstimatedStructSize();
-  if (query_cache_context->estimated_out_bytes > max_query_size_bytes_) {
-    std::move(query_cache_context->callback)
-        .Run(CACHE_STORAGE_ERROR_QUERY_TOO_LARGE,
-             std::unique_ptr<QueryCacheResults>());
-    return;
+  if (query_cache_context->query_types & QUERY_CACHE_RESPONSES_WITH_BODIES) {
+    query_cache_context->estimated_out_bytes +=
+        match->response->EstimatedStructSize();
+    if (query_cache_context->estimated_out_bytes > max_query_size_bytes_) {
+      std::move(query_cache_context->callback)
+          .Run(CACHE_STORAGE_ERROR_QUERY_TOO_LARGE,
+               std::unique_ptr<QueryCacheResults>());
+      return;
+    }
+    if (entry->GetDataSize(INDEX_RESPONSE_BODY) == 0) {
+      QueryCacheOpenNextEntry(std::move(query_cache_context));
+      return;
+    }
+
+    if (!blob_storage_context_) {
+      std::move(query_cache_context->callback)
+          .Run(CACHE_STORAGE_ERROR_STORAGE,
+               std::unique_ptr<QueryCacheResults>());
+      return;
+    }
+
+    std::unique_ptr<storage::BlobDataHandle> blob_data_handle =
+        PopulateResponseBody(std::move(entry), match->response.get());
+    match->blob_handle = std::move(blob_data_handle);

[jkarlin, 2017/07/21 21:18:08]

Seems like we could combine the above lines:

match->blob_handle = PopulateResponseBody(std::move(entry),
match->response.get());

[cmumford, 2017/07/27 23:33:24]

Done.

+  } else if (!(query_cache_context->query_types &
+               QUERY_CACHE_RESPONSES_NO_BODIES)) {
+    match->response.reset();
   }
 
-  if (query_cache_context->query_type == QueryCacheType::REQUESTS) {
-    match->response.reset();
-    QueryCacheOpenNextEntry(std::move(query_cache_context));
-    return;
-  }
-
-  DCHECK_EQ(QueryCacheType::REQUESTS_AND_RESPONSES,
-            query_cache_context->query_type);
-
-  query_cache_context->estimated_out_bytes +=
-      match->response->EstimatedStructSize();
-  if (query_cache_context->estimated_out_bytes > max_query_size_bytes_) {
-    std::move(query_cache_context->callback)
-        .Run(CACHE_STORAGE_ERROR_QUERY_TOO_LARGE,
-             std::unique_ptr<QueryCacheResults>());
-    return;
-  }
-
-  if (entry->GetDataSize(INDEX_RESPONSE_BODY) == 0) {
-    QueryCacheOpenNextEntry(std::move(query_cache_context));
-    return;
-  }
-
-  if (!blob_storage_context_) {
-    std::move(query_cache_context->callback)
-        .Run(CACHE_STORAGE_ERROR_STORAGE,
-             base::MakeUnique<QueryCacheResults>());
-    return;
-  }
-
-  std::unique_ptr<storage::BlobDataHandle> blob_data_handle =
-      PopulateResponseBody(std::move(entry), match->response.get());
-  match->blob_handle = std::move(blob_data_handle);
-
   QueryCacheOpenNextEntry(std::move(query_cache_context));
 }
 
 // static
 bool CacheStorageCache::QueryCacheResultCompare(const QueryCacheResult& lhs,
                                                 const QueryCacheResult& rhs) {
   return lhs.entry_time < rhs.entry_time;
 }
 
+// static
+int64_t CacheStorageCache::CalculateResponsePadding(
+    const ServiceWorkerResponse& response,
+    const crypto::SymmetricKey* padding_key,
+    int side_data_size) {
+  if (!ShouldPadResourceSize(&response))
+    return 0;
+  return CalculateResponsePaddingInternal(
+      &response, response.url_list.back().spec(), padding_key, side_data_size);
+}
+
+// static
+int32_t CacheStorageCache::GetResponsePaddingVersion() {
+  return kCachePaddingAlgorithmVersion;
+}
+
 void CacheStorageCache::MatchImpl(
     std::unique_ptr<ServiceWorkerFetchRequest> request,
     const CacheStorageCacheQueryParams& match_params,
     ResponseCallback callback) {
   MatchAllImpl(
       std::move(request), match_params,
       base::BindOnce(&CacheStorageCache::MatchDidMatchAll,
                      weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
 }
 
@@ skipping 29 matching lines @@
     ResponsesCallback callback) {
   DCHECK_NE(BACKEND_UNINITIALIZED, backend_state_);
   if (backend_state_ != BACKEND_OPEN) {
     std::move(callback).Run(CACHE_STORAGE_ERROR_STORAGE,
                             std::unique_ptr<Responses>(),
                             std::unique_ptr<BlobDataHandles>());
     return;
   }
 
   QueryCache(
-      std::move(request), options, QueryCacheType::REQUESTS_AND_RESPONSES,
+      std::move(request), options,
+      QUERY_CACHE_REQUESTS | QUERY_CACHE_RESPONSES_WITH_BODIES,
       base::BindOnce(&CacheStorageCache::MatchAllDidQueryCache,
                      weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
 }
 
 void CacheStorageCache::MatchAllDidQueryCache(
     ResponsesCallback callback,
     CacheStorageError error,
     std::unique_ptr<QueryCacheResults> query_cache_results) {
   if (error != CACHE_STORAGE_OK) {
     std::move(callback).Run(error, std::unique_ptr<Responses>(),
@@ skipping 92 matching lines @@
     std::unique_ptr<proto::CacheMetadata> headers) {
   if (!headers ||
       headers->response().response_time() !=
           expected_response_time.ToInternalValue()) {
     std::move(callback).Run(CACHE_STORAGE_ERROR_NOT_FOUND);
     return;
   }
   // Get a temporary copy of the entry pointer before passing it in base::Bind.
   disk_cache::Entry* temp_entry_ptr = entry.get();
 
+  std::unique_ptr<content::proto::CacheResponse> response(
+      headers->release_response());
+
+  int side_data_size_before_write = 0;
+  if (ShouldPadResourceSize(response.get()))
+    side_data_size_before_write = entry->GetDataSize(INDEX_SIDE_DATA);
+
   net::CompletionCallback write_side_data_callback =
       base::AdaptCallbackForRepeating(
           base::BindOnce(&CacheStorageCache::WriteSideDataDidWrite,
                          weak_ptr_factory_.GetWeakPtr(), std::move(callback),
-                         base::Passed(std::move(entry)), buf_len));
+                         base::Passed(std::move(entry)), buf_len,
+                         std::move(response), side_data_size_before_write));
 
   int rv = temp_entry_ptr->WriteData(
       INDEX_SIDE_DATA, 0 /* offset */, buffer.get(), buf_len,
       write_side_data_callback, true /* truncate */);
 
   if (rv != net::ERR_IO_PENDING)
     write_side_data_callback.Run(rv);
 }
 
-void CacheStorageCache::WriteSideDataDidWrite(ErrorCallback callback,
-                                              disk_cache::ScopedEntryPtr entry,
-                                              int expected_bytes,
-                                              int rv) {
+void CacheStorageCache::WriteSideDataDidWrite(
+    ErrorCallback callback,
+    disk_cache::ScopedEntryPtr entry,
+    int expected_bytes,
+    std::unique_ptr<::content::proto::CacheResponse> response,
+    int side_data_size_before_write,
+    int rv) {
   if (rv != expected_bytes) {
     entry->Doom();
     UpdateCacheSize(
         base::BindOnce(std::move(callback), CACHE_STORAGE_ERROR_NOT_FOUND));
     return;
   }
 
   if (rv > 0)
     storage::RecordBytesWritten(kRecordBytesLabel, rv);
 
+  if (ShouldPadResourceSize(response.get())) {
+    cache_padding_ -= CalculateResponsePaddingInternal(
+        response.get(), cache_padding_key_.get(), side_data_size_before_write);

[jkarlin, 2017/07/21 21:18:09]

Let's add a test where we write side data twice (each with a different size) and
verify that the padded cache size is the same (delta the difference in side data
size).

[cmumford, 2017/07/27 23:33:24]

Done: TestDifferentOpaqueSideDataSizes.

+
+    cache_padding_ += CalculateResponsePaddingInternal(
+        response.get(), cache_padding_key_.get(), rv);
+  }
+
   UpdateCacheSize(base::BindOnce(std::move(callback), CACHE_STORAGE_OK));
 }
 
 void CacheStorageCache::Put(const CacheStorageBatchOperation& operation,
                             ErrorCallback callback) {
   DCHECK(BACKEND_OPEN == backend_state_ || initializing_);
   DCHECK_EQ(CACHE_STORAGE_CACHE_OPERATION_TYPE_PUT, operation.operation_type);
 
   std::unique_ptr<ServiceWorkerFetchRequest> request(
       new ServiceWorkerFetchRequest(
@@ skipping 33 matching lines @@
       base::Passed(std::move(put_context))));
 }
 
 void CacheStorageCache::PutImpl(std::unique_ptr<PutContext> put_context) {
   DCHECK_NE(BACKEND_UNINITIALIZED, backend_state_);
   if (backend_state_ != BACKEND_OPEN) {
     std::move(put_context->callback).Run(CACHE_STORAGE_ERROR_STORAGE);
     return;
   }
 
-  std::string key = put_context->request->url.spec();
+  // Explicitly delete the incumbent resource (which may not exist). This is
+  // only done so that it's padding will be decremented from the calculated
+  // cache padding.
+  // TODO(cmumford): Research alternatives to this explicit delete as it
+  // seriously impacts put performance.
+  auto delete_request = base::MakeUnique<ServiceWorkerFetchRequest>(
+      put_context->request->url, "", ServiceWorkerHeaderMap(), Referrer(),
+      false);
 
-  net::CompletionCallback callback =
-      base::AdaptCallbackForRepeating(base::BindOnce(
-          &CacheStorageCache::PutDidDoomEntry, weak_ptr_factory_.GetWeakPtr(),
-          base::Passed(std::move(put_context))));
-
-  int rv = backend_->DoomEntry(key, callback);
-  if (rv != net::ERR_IO_PENDING)
-    callback.Run(rv);
+  CacheStorageCacheQueryParams query_options;
+  query_options.ignore_method = true;
+  query_options.ignore_vary = true;
+  DeleteImpl(std::move(delete_request), query_options,
+             base::BindOnce(&CacheStorageCache::PutDidDeleteEntry,
+                            weak_ptr_factory_.GetWeakPtr(),
+                            base::Passed(std::move(put_context))));
 }
 
-void CacheStorageCache::PutDidDoomEntry(std::unique_ptr<PutContext> put_context,
-                                        int rv) {
+void CacheStorageCache::PutDidDeleteEntry(
+    std::unique_ptr<PutContext> put_context,
+    CacheStorageError error,
+    std::unique_ptr<QueryCacheResults> query_results) {
   if (backend_state_ != BACKEND_OPEN) {
     std::move(put_context->callback).Run(CACHE_STORAGE_ERROR_STORAGE);
     return;
   }
 
-  // |rv| is ignored as doom entry can fail if the entry doesn't exist.
+  for (const auto& result : *query_results) {
+    if (ShouldPadResourceSize(result.response.get())) {
+      cache_padding_ -=

[jkarlin, 2017/07/21 21:18:09]

This still confuses me. DeleteImpl() already alters cache_padding_ in
DeleteDidQueryCache for the deleted results, why are we doing it again here? And
if it's not needed, we can remove the new code where Delete returns entries.

Let's add a test that shows putting the same entry in twice results in the same
cache size after eat put operation.

[cmumford, 2017/07/27 23:33:24]

Yes, you were correct. I added TestDoubleOpaquePut and deleted this code.

+          CalculateResponsePadding(*result.response, cache_padding_key_.get(),
+                                   result.entry->GetDataSize(INDEX_SIDE_DATA));
+    }
+  }
+
+  if (error != CACHE_STORAGE_OK && error != CACHE_STORAGE_ERROR_NOT_FOUND) {
+    std::move(put_context->callback).Run(error);
+    return;
+  }
 
   std::unique_ptr<disk_cache::Entry*> scoped_entry_ptr(
       new disk_cache::Entry*());
   disk_cache::Entry** entry_ptr = scoped_entry_ptr.get();
   ServiceWorkerFetchRequest* request_ptr = put_context->request.get();
   disk_cache::Backend* backend_ptr = backend_.get();
 
   net::CompletionCallback create_entry_callback =
       base::AdaptCallbackForRepeating(base::BindOnce(
           &CacheStorageCache::PutDidCreateEntry, weak_ptr_factory_.GetWeakPtr(),
           base::Passed(std::move(scoped_entry_ptr)),
           base::Passed(std::move(put_context))));
 
-  int create_rv = backend_ptr->CreateEntry(request_ptr->url.spec(), entry_ptr,
-                                           create_entry_callback);
+  int rv = backend_ptr->CreateEntry(request_ptr->url.spec(), entry_ptr,
+                                    create_entry_callback);
 
-  if (create_rv != net::ERR_IO_PENDING)
-    create_entry_callback.Run(create_rv);
+  if (rv != net::ERR_IO_PENDING)
+    create_entry_callback.Run(rv);
 }
 
 void CacheStorageCache::PutDidCreateEntry(
     std::unique_ptr<disk_cache::Entry*> entry_ptr,
     std::unique_ptr<PutContext> put_context,
     int rv) {
   put_context->cache_entry.reset(*entry_ptr);
 
   if (rv != net::OK) {
     std::move(put_context->callback).Run(CACHE_STORAGE_ERROR_EXISTS);
@@ skipping 66 matching lines @@
     int expected_bytes,
     int rv) {
   if (rv != expected_bytes) {
     put_context->cache_entry->Doom();
     std::move(put_context->callback).Run(CACHE_STORAGE_ERROR_STORAGE);
     return;
   }
 
   if (rv > 0)
     storage::RecordBytesWritten(kRecordBytesLabel, rv);
+  if (ShouldPadResourceSize(put_context->response.get())) {
+    cache_padding_ += CalculateResponsePadding(*put_context->response,
+                                               cache_padding_key_.get(),
+                                               0 /* side_data_size */);
+  }
 
   // The metadata is written, now for the response content. The data is streamed
   // from the blob into the cache entry.
 
   if (put_context->response->blob_uuid.empty()) {
     UpdateCacheSize(
         base::BindOnce(std::move(put_context->callback), CACHE_STORAGE_OK));
     return;
   }
 
@@ skipping 31 matching lines @@
   if (!success) {
     put_context->cache_entry->Doom();
     std::move(put_context->callback).Run(CACHE_STORAGE_ERROR_STORAGE);
     return;
   }
 
   UpdateCacheSize(
       base::BindOnce(std::move(put_context->callback), CACHE_STORAGE_OK));
 }
 
+void CacheStorageCache::CalculateCacheSizePadding(
+    SizePaddingCallback got_sizes_callback) {
+  net::CompletionCallback got_size_callback =
+      base::AdaptCallbackForRepeating(base::BindOnce(
+          &CacheStorageCache::CalculateCacheSizePaddingGotSize,
+          weak_ptr_factory_.GetWeakPtr(), std::move(got_sizes_callback)));
+
+  int rv = backend_->CalculateSizeOfAllEntries(got_size_callback);
+  if (rv != net::ERR_IO_PENDING)
+    got_size_callback.Run(rv);
+}
+

[jkarlin, 2017/07/21 21:18:08]

Please update the README.md for this directory with a description of padding and
how it works into the architecture

[cmumford, 2017/07/27 23:33:24]

Done.

+void CacheStorageCache::CalculateCacheSizePaddingGotSize(
+    SizePaddingCallback callback,
+    int cache_size) {
+  // Enumerating entries is only done during cache initialization.

[jkarlin, 2017/07/21 21:18:08]

only done during cache initialization and only if necessary

[cmumford, 2017/07/27 23:33:24]

Done.

+  DCHECK_EQ(backend_state_, BACKEND_UNINITIALIZED);
+  std::unique_ptr<ServiceWorkerFetchRequest> request;
+  CacheStorageCacheQueryParams options;
+  options.ignore_search = true;
+  QueryCache(std::move(request), options, QUERY_CACHE_RESPONSES_NO_BODIES,
+             base::BindOnce(&CacheStorageCache::PaddingDidQueryCache,
+                            weak_ptr_factory_.GetWeakPtr(), std::move(callback),
+                            cache_size));
+}
+
+void CacheStorageCache::PaddingDidQueryCache(
+    SizePaddingCallback callback,
+    int cache_size,
+    CacheStorageError error,
+    std::unique_ptr<QueryCacheResults> query_cache_results) {
+  int64_t cache_padding = 0;
+  if (error == CACHE_STORAGE_OK) {
+    for (const auto& result : *query_cache_results) {
+      if (ShouldPadResourceSize(result.response.get())) {
+        cache_padding += CalculateResponsePadding(
+            *result.response, cache_padding_key_.get(),
+            result.entry->GetDataSize(INDEX_SIDE_DATA));
+      }
+    }
+  }
+
+  std::move(callback).Run(cache_size, cache_padding);
+}
+
+void CacheStorageCache::CalculateCacheSize(
+    const net::CompletionCallback& callback) {
+  int rv = backend_->CalculateSizeOfAllEntries(callback);
+  if (rv != net::ERR_IO_PENDING)
+    callback.Run(rv);
+}
+
 void CacheStorageCache::UpdateCacheSize(base::OnceClosure callback) {
   if (backend_state_ != BACKEND_OPEN)
     return;
 
   // Note that the callback holds a cache handle to keep the cache alive during
   // the operation since this UpdateCacheSize is often run after an operation
   // completes and runs its callback.
-  net::CompletionCallback size_callback = base::AdaptCallbackForRepeating(
-      base::BindOnce(&CacheStorageCache::UpdateCacheSizeGotSize,
-                     weak_ptr_factory_.GetWeakPtr(),
-                     base::Passed(CreateCacheHandle()), std::move(callback)));
-
-  int rv = backend_->CalculateSizeOfAllEntries(size_callback);
-  if (rv != net::ERR_IO_PENDING)
-    std::move(size_callback).Run(rv);
+  net::CompletionCallback update_size_callback =
+      base::AdaptCallbackForRepeating(base::BindOnce(
+          &CacheStorageCache::UpdateCacheSizeGotSize,
+          weak_ptr_factory_.GetWeakPtr(), base::Passed(CreateCacheHandle()),
+          std::move(callback)));
+  CalculateCacheSize(update_size_callback);
 }
 
 void CacheStorageCache::UpdateCacheSizeGotSize(
     std::unique_ptr<CacheStorageCacheHandle> cache_handle,
     base::OnceClosure callback,
     int current_cache_size) {
   DCHECK_NE(current_cache_size, CacheStorage::kSizeUnknown);
-  int64_t old_cache_size = cache_size_;
   cache_size_ = current_cache_size;
-
-  int64_t size_delta = current_cache_size - old_cache_size;
+  int64_t size_delta = PaddedCacheSize() - last_reported_size_;

[jkarlin, 2017/07/21 21:18:08]

last_reported_size_ initializes to 0 and doesn't get updated before this point.
Which means after loading a preexisting cache, the first time you change
something you're going to have the wrong size_delta and the QuotaManager will
have the wrong size.

Please fix and add relevant test. I believe we have some tests that measure
notifystoragemodified calls.

[cmumford, 2017/07/27 23:33:24]

I added CacheStorageManagerTest.CacheSizePaddedAfterReopen which was a needed
test - thx. However, I'm not sure I'm seeing the problem you describe.

+  last_reported_size_ = PaddedCacheSize();
 
   quota_manager_proxy_->NotifyStorageModified(
       storage::QuotaClient::kServiceWorkerCache, origin_,
       storage::kStorageTypeTemporary, size_delta);
 
   if (cache_observer_)
-    cache_observer_->CacheSizeUpdated(this, current_cache_size);
+    cache_observer_->CacheSizeUpdated(this, PaddedCacheSize());
 
   std::move(callback).Run();
 }
 
 void CacheStorageCache::Delete(const CacheStorageBatchOperation& operation,
                                ErrorCallback callback) {
   DCHECK(BACKEND_OPEN == backend_state_ || initializing_);
   DCHECK_EQ(CACHE_STORAGE_CACHE_OPERATION_TYPE_DELETE,
             operation.operation_type);
 
   std::unique_ptr<ServiceWorkerFetchRequest> request(
       new ServiceWorkerFetchRequest(
           operation.request.url, operation.request.method,
           operation.request.headers, operation.request.referrer,
           operation.request.is_reload));
 
+  QueryCacheCallback delete_callback =
+      base::BindOnce(&CacheStorageCache::DeleteDidDelete,
+                     weak_ptr_factory_.GetWeakPtr(), std::move(callback));
+
   scheduler_->ScheduleOperation(base::BindOnce(
       &CacheStorageCache::DeleteImpl, weak_ptr_factory_.GetWeakPtr(),
       base::Passed(std::move(request)), operation.match_params,
-      scheduler_->WrapCallbackToRunNext(std::move(callback))));
+      scheduler_->WrapCallbackToRunNext(std::move(delete_callback))));
+}
+
+void CacheStorageCache::DeleteDidDelete(

[jkarlin, 2017/07/21 21:18:09]

Can you add a comment specifying that the purpose of this function is to ignore
the query results?

[jkarlin, 2017/07/21 21:18:09]

This function isn't in the same order as the header and needs to be moved down
below DeleteDidQueryCache.

[cmumford, 2017/07/27 23:33:24]

Done.

[cmumford, 2017/07/27 23:33:24]

Done.

+    ErrorCallback callback,
+    CacheStorageError error,
+    std::unique_ptr<QueryCacheResults> query_results) {
+  std::move(callback).Run(error);
 }
 
 void CacheStorageCache::DeleteImpl(
     std::unique_ptr<ServiceWorkerFetchRequest> request,
     const CacheStorageCacheQueryParams& match_params,
-    ErrorCallback callback) {
+    QueryCacheCallback callback) {
   DCHECK_NE(BACKEND_UNINITIALIZED, backend_state_);
   if (backend_state_ != BACKEND_OPEN) {
-    std::move(callback).Run(CACHE_STORAGE_ERROR_STORAGE);
+    std::move(callback).Run(CACHE_STORAGE_ERROR_STORAGE,
+                            base::MakeUnique<QueryCacheResults>());
     return;
   }
 
   QueryCache(
-      std::move(request), match_params, QueryCacheType::CACHE_ENTRIES,
+      std::move(request), match_params,
+      QUERY_CACHE_ENTRIES | QUERY_CACHE_RESPONSES_NO_BODIES,
       base::BindOnce(&CacheStorageCache::DeleteDidQueryCache,
                      weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
 }
 
 void CacheStorageCache::DeleteDidQueryCache(
-    ErrorCallback callback,
+    QueryCacheCallback callback,
     CacheStorageError error,
     std::unique_ptr<QueryCacheResults> query_cache_results) {
   if (error != CACHE_STORAGE_OK) {
-    std::move(callback).Run(error);
+    std::move(callback).Run(error, std::move(query_cache_results));
     return;
   }
 
   if (query_cache_results->empty()) {
-    std::move(callback).Run(CACHE_STORAGE_ERROR_NOT_FOUND);
+    std::move(callback).Run(CACHE_STORAGE_ERROR_NOT_FOUND,
+                            std::move(query_cache_results));
     return;
   }
 
   for (auto& result : *query_cache_results) {
     disk_cache::ScopedEntryPtr entry = std::move(result.entry);
+    if (ShouldPadResourceSize(result.response.get())) {
+      cache_padding_ -=
+          CalculateResponsePadding(*result.response, cache_padding_key_.get(),
+                                   entry->GetDataSize(INDEX_SIDE_DATA));
+    }
     entry->Doom();
   }
 
-  UpdateCacheSize(base::BindOnce(std::move(callback), CACHE_STORAGE_OK));
+  UpdateCacheSize(base::BindOnce(std::move(callback), CACHE_STORAGE_OK,
+                                 base::Passed(std::move(query_cache_results))));
 }
 
 void CacheStorageCache::KeysImpl(
     std::unique_ptr<ServiceWorkerFetchRequest> request,
     const CacheStorageCacheQueryParams& options,
     RequestsCallback callback) {
   DCHECK_NE(BACKEND_UNINITIALIZED, backend_state_);
   if (backend_state_ != BACKEND_OPEN) {
     std::move(callback).Run(CACHE_STORAGE_ERROR_STORAGE,
                             std::unique_ptr<Requests>());
     return;
   }
 
   QueryCache(
-      std::move(request), options, QueryCacheType::REQUESTS,
+      std::move(request), options, QUERY_CACHE_REQUESTS,
       base::BindOnce(&CacheStorageCache::KeysDidQueryCache,
                      weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
 }
 
 void CacheStorageCache::KeysDidQueryCache(
     RequestsCallback callback,
     CacheStorageError error,
     std::unique_ptr<QueryCacheResults> query_cache_results) {
   if (error != CACHE_STORAGE_OK) {
     std::move(callback).Run(error, std::unique_ptr<Requests>());
@@ skipping 12 matching lines @@
   DCHECK_NE(BACKEND_CLOSED, backend_state_);
 
   backend_state_ = BACKEND_CLOSED;
   backend_.reset();
   std::move(callback).Run();
 }
 
 void CacheStorageCache::SizeImpl(SizeCallback callback) {
   DCHECK_NE(BACKEND_UNINITIALIZED, backend_state_);
 
-  int64_t size = backend_state_ == BACKEND_OPEN ? cache_size_ : 0;
+  // TODO(cmumford): Can CacheStorage::kSizeUnknown be returned instead of zero?
+  if (backend_state_ != BACKEND_OPEN) {
+    base::ThreadTaskRunnerHandle::Get()->PostTask(
+        FROM_HERE, base::BindOnce(std::move(callback), 0));
+    return;
+  }
+
+  int64_t size = backend_state_ == BACKEND_OPEN ? PaddedCacheSize() : 0;
   base::ThreadTaskRunnerHandle::Get()->PostTask(
       FROM_HERE, base::BindOnce(std::move(callback), size));
 }
 
 void CacheStorageCache::GetSizeThenCloseDidGetSize(SizeCallback callback,
                                                    int64_t cache_size) {
   CloseImpl(base::BindOnce(std::move(callback), cache_size));
 }
 
 void CacheStorageCache::CreateBackend(ErrorCallback callback) {
@@ skipping 52 matching lines @@
 }
 
 void CacheStorageCache::InitDidCreateBackend(
     base::OnceClosure callback,
     CacheStorageError cache_create_error) {
   if (cache_create_error != CACHE_STORAGE_OK) {
     InitGotCacheSize(std::move(callback), cache_create_error, 0);
     return;
   }
 
-  net::CompletionCallback size_callback =
-      base::AdaptCallbackForRepeating(base::BindOnce(
-          &CacheStorageCache::InitGotCacheSize, weak_ptr_factory_.GetWeakPtr(),
-          std::move(callback), cache_create_error));
+  auto calculate_size_callback =
+      base::AdaptCallbackForRepeating(std::move(callback));
+  int rv = backend_->CalculateSizeOfAllEntries(base::Bind(
+      &CacheStorageCache::InitGotCacheSize, weak_ptr_factory_.GetWeakPtr(),
+      calculate_size_callback, cache_create_error));
 
-  int rv = backend_->CalculateSizeOfAllEntries(size_callback);
   if (rv != net::ERR_IO_PENDING)
-    std::move(size_callback).Run(rv);
+    InitGotCacheSize(calculate_size_callback, cache_create_error, rv);
 }
 
 void CacheStorageCache::InitGotCacheSize(base::OnceClosure callback,
                                          CacheStorageError cache_create_error,
                                          int cache_size) {
   // Now that we know the cache size either 1) the cache size should be unknown
   // (which is why the size was calculated), or 2) it must match the current
   // size. If the sizes aren't equal then there is a bug in how the cache size
   // is saved in the store's index.
   if (cache_size_ != CacheStorage::kSizeUnknown) {
     DLOG_IF(ERROR, cache_size_ != cache_size)
         << "Cache size: " << cache_size
         << " does not match size from index: " << cache_size_;
     UMA_HISTOGRAM_COUNTS_10M("ServiceWorkerCache.IndexSizeDifference",
                              std::abs(cache_size_ - cache_size));
-    // Disabled for crbug.com/681900.
-    // DCHECK_EQ(cache_size_, cache_size);
+    if (cache_size_ != cache_size) {
+      // We assume that if the sizes match then then cached padding is still
+      // correct. If not then we recalculate the padding.
+      CalculateCacheSizePaddingGotSize(
+          base::BindOnce(&CacheStorageCache::InitGotCacheSizeAndPadding,
+                         weak_ptr_factory_.GetWeakPtr(), std::move(callback),
+                         cache_create_error),
+          cache_size);
+      return;
+    }
   }
+
+  if (cache_padding_ == CacheStorage::kSizeUnknown || cache_padding_ < 0) {
+    CalculateCacheSizePaddingGotSize(
+        base::BindOnce(&CacheStorageCache::InitGotCacheSizeAndPadding,
+                       weak_ptr_factory_.GetWeakPtr(), std::move(callback),
+                       cache_create_error),
+        cache_size);
+    return;
+  }
+
+  // If cached size matches actual size then assume cached padding is still
+  // correct.
+  InitGotCacheSizeAndPadding(std::move(callback), cache_create_error,
+                             cache_size, cache_padding_);
+}
+
+void CacheStorageCache::InitGotCacheSizeAndPadding(
+    base::OnceClosure callback,
+    CacheStorageError cache_create_error,
+    int64_t cache_size,
+    int64_t cache_padding) {
   cache_size_ = cache_size;
+  cache_padding_ = cache_padding;
+
   initializing_ = false;
   backend_state_ = (cache_create_error == CACHE_STORAGE_OK && backend_ &&
                     backend_state_ == BACKEND_UNINITIALIZED)
                        ? BACKEND_OPEN
                        : BACKEND_CLOSED;
 
   UMA_HISTOGRAM_ENUMERATION("ServiceWorkerCache.InitBackendResult",
                             cache_create_error, CACHE_STORAGE_ERROR_LAST + 1);
 
   if (cache_observer_)
-    cache_observer_->CacheSizeUpdated(this, cache_size_);
+    cache_observer_->CacheSizeUpdated(this, PaddedCacheSize());
 
   std::move(callback).Run();
 }
 
 std::unique_ptr<storage::BlobDataHandle>
 CacheStorageCache::PopulateResponseBody(disk_cache::ScopedEntryPtr entry,
                                         ServiceWorkerResponse* response) {
   DCHECK(blob_storage_context_);
 
   // Create a blob with the response body data.
   response->blob_size = entry->GetDataSize(INDEX_RESPONSE_BODY);
   response->blob_uuid = base::GenerateGUID();
   storage::BlobDataBuilder blob_data(response->blob_uuid);
 
   disk_cache::Entry* temp_entry = entry.get();
   blob_data.AppendDiskCacheEntryWithSideData(
       new CacheStorageCacheDataHandle(CreateCacheHandle(), std::move(entry)),
       temp_entry, INDEX_RESPONSE_BODY, INDEX_SIDE_DATA);
   return blob_storage_context_->AddFinishedBlob(&blob_data);
 }
 
 std::unique_ptr<CacheStorageCacheHandle>
 CacheStorageCache::CreateCacheHandle() {
   return cache_storage_->CreateCacheHandle(this);
 }
 
+int64_t CacheStorageCache::PaddedCacheSize() const {
+  if (cache_size_ == CacheStorage::kSizeUnknown ||
+      cache_padding_ == CacheStorage::kSizeUnknown) {

[jkarlin, 2017/07/21 21:18:08]

Can we DCHECK that the cache hasn't initialized yet if we're in this state?

[cmumford, 2017/07/27 23:33:24]

Done.

+    return CacheStorage::kSizeUnknown;
+  }
+  return cache_size_ + cache_padding_;
+}
+
 }  // namespace content