Separate preload matching from MemoryCache

third_party/WebKit/Source/platform/loader/fetch/ResourceFetcher.cpp

 /*
     Copyright (C) 1998 Lars Knoll (knoll@mpi-hd.mpg.de)
     Copyright (C) 2001 Dirk Mueller (mueller@kde.org)
     Copyright (C) 2002 Waldo Bastian (bastian@kde.org)
     Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Apple Inc. All
     rights reserved.
     Copyright (C) 2009 Torch Mobile Inc. http://www.torchmobile.com/
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Library General Public
@@ skipping 475 matching lines @@
   if (params.IsSpeculativePreload() || params.IsLinkPreload()) {
     DEFINE_RESOURCE_HISTOGRAM("Preload.");
   } else {
     DEFINE_RESOURCE_HISTOGRAM("");
   }
 
   // Aims to count Resource only referenced from MemoryCache (i.e. what would be
   // dead if MemoryCache holds weak references to Resource). Currently we check
   // references to Resource from ResourceClient and |m_preloads| only, because
   // they are major sources of references.
-  if (resource && !resource->IsAlive() && !preloads_.Contains(resource)) {
+  if (resource && !resource->IsAlive() && !ContainsAsPreload(resource)) {
     DEFINE_RESOURCE_HISTOGRAM("Dead.");
   }
 }
 
+bool ResourceFetcher::ContainsAsPreload(Resource* resource) const {
+  auto it = preloads_.find(PreloadKey(resource->Url(), resource->GetType()));
+  return it != preloads_.end() && it->value == resource;
+}
+
+void ResourceFetcher::RemovePreload(Resource* resource) {
+  auto it = preloads_.find(PreloadKey(resource->Url(), resource->GetType()));
+  if (it == preloads_.end())
+    return;
+  if (it->value == resource) {
+    resource->DecreasePreloadCount();
+    preloads_.erase(it);
+  }
+}
+
 ResourceFetcher::PrepareRequestResult ResourceFetcher::PrepareRequest(
     FetchParameters& params,
     const ResourceFactory& factory,
     const SubstituteData& substitute_data,
     unsigned long identifier,
     ResourceRequestBlockedReason& blocked_reason) {
   ResourceRequest& resource_request = params.MutableResourceRequest();
 
   DCHECK(params.Options().synchronous_policy == kRequestAsynchronously ||
          factory.GetType() == Resource::kRaw ||
@@ skipping 86 matching lines @@
   bool is_static_data = is_data_url || substitute_data.IsValid() || archive_;
   if (is_static_data) {
     resource = ResourceForStaticData(params, factory, substitute_data);
     // Abort the request if the archive doesn't contain the resource, except in
     // the case of data URLs which might have resources such as fonts that need
     // to be decoded only on demand. These data URLs are allowed to be
     // processed using the normal ResourceFetcher machinery.
     if (!resource && !is_data_url && archive_)
       return nullptr;
   }
+  RevalidationPolicy policy = kLoad;
+  bool preload_found = false;
   if (!resource) {
-    resource =
-        GetMemoryCache()->ResourceForURL(params.Url(), GetCacheIdentifier());
+    resource = MatchPreload(params, factory.GetType());
+    if (resource) {
+      preload_found = true;
+      policy = kUse;
+      // If |param| is for a blocking resource and a preloaded resource is
+      // found, we may need to make it block the onload event.
+      MakePreloadedResourceBlockOnloadIfNeeded(resource, params);
+    }
   }
+  if (!preload_found) {
+    if (!resource) {
+      resource =
+          GetMemoryCache()->ResourceForURL(params.Url(), GetCacheIdentifier());
+    }
 
-  // If we got a preloaded resource from the cache for a non-preload request,
-  // we may need to make it block the onload event.
-  MakePreloadedResourceBlockOnloadIfNeeded(resource, params);
-
-  const RevalidationPolicy policy = DetermineRevalidationPolicy(
-      factory.GetType(), params, resource, is_static_data);
-  TRACE_EVENT_INSTANT1("blink", "ResourceFetcher::determineRevalidationPolicy",
-                       TRACE_EVENT_SCOPE_THREAD, "revalidationPolicy", policy);
+    policy = DetermineRevalidationPolicy(factory.GetType(), params, resource,
+                                         is_static_data);
+    TRACE_EVENT_INSTANT1(
+        "blink", "ResourceFetcher::determineRevalidationPolicy",
+        TRACE_EVENT_SCOPE_THREAD, "revalidationPolicy", policy);
+  }
 
   UpdateMemoryCacheStats(resource, policy, params, factory, is_static_data);
 
   switch (policy) {
     case kReload:
       GetMemoryCache()->Remove(resource);
     // Fall through
     case kLoad:
       resource = CreateResourceForLoading(params, params.Charset(), factory);
       break;
@@ skipping 32 matching lines @@
   // If only the fragment identifiers differ, it is the same resource.
   DCHECK(EqualIgnoringFragmentIdentifier(resource->Url(), params.Url()));
   RequestLoadStarted(identifier, resource, params, policy, is_static_data);
   document_resources_.Set(
       MemoryCache::RemoveFragmentIdentifierIfNeeded(params.Url()), resource);
 
   // Returns with an existing resource if the resource does not need to start
   // loading immediately. If revalidation policy was determined as |Revalidate|,
   // the resource was already initialized for the revalidation here, but won't
   // start loading.
-  if (!ResourceNeedsLoad(resource, params, policy))
+  if (!ResourceNeedsLoad(resource, params, policy)) {
+    if (policy != kUse)
+      InsertAsPreloadIfNecessary(resource, params, factory.GetType());
     return resource;
+  }
 
   if (!StartLoad(resource))
     return nullptr;
+
+  if (policy != kUse)
+    InsertAsPreloadIfNecessary(resource, params, factory.GetType());
   scoped_resource_load_tracker.ResourceLoadContinuesBeyondScope();
 
   DCHECK(!resource->ErrorOccurred() ||
          params.Options().synchronous_policy == kRequestSynchronously);
   return resource;
 }
 
 void ResourceFetcher::ResourceTimingReportTimerFired(TimerBase* timer) {
   DCHECK_EQ(timer, &resource_timing_report_timer_);
   Vector<RefPtr<ResourceTimingInfo>> timing_reports;
@@ skipping 75 matching lines @@
   resource->SetLinkPreload(params.IsLinkPreload());
   if (params.IsSpeculativePreload()) {
     resource->SetPreloadDiscoveryTime(params.PreloadDiscoveryTime());
   }
   resource->SetCacheIdentifier(cache_identifier);
 
   // - Don't add main resource to cache to prevent reuse.
   // - Don't add the resource if its body will not be stored.
   if (factory.GetType() != Resource::kMainResource &&
       params.Options().data_buffering_policy != kDoNotBufferData) {
     GetMemoryCache()->Add(resource);

[Yoav Weiss, 2017/05/12 16:53:46]

Won't this add preloaded resources also to the memoryCache?

[yhirano, 2017/05/16 11:19:24]

Yes, this CL changes how to match preloads, and doesn't change how to store
resources to MemoryCache.

   }
   return resource;
 }
 
 void ResourceFetcher::StorePerformanceTimingInitiatorInformation(
     Resource* resource) {
   const AtomicString& fetch_initiator = resource->Options().initiator_info.name;
   if (fetch_initiator == FetchInitiatorTypeNames::internal)
     return;
 
@@ skipping 37 matching lines @@
   if (it != resource_timing_info_map_.end()) {
     it->value->AddRedirect(redirect_response, cross_origin);
   }
 
   if (resource->GetType() == Resource::kMainResource) {
     DCHECK(navigation_timing_info_);
     navigation_timing_info_->AddRedirect(redirect_response, cross_origin);
   }
 }
 
-ResourceFetcher::RevalidationPolicy
-ResourceFetcher::DetermineRevalidationPolicy(
-    Resource::Type type,
-    const FetchParameters& fetch_params,
-    Resource* existing_resource,
-    bool is_static_data) const {
-  const ResourceRequest& request = fetch_params.GetResourceRequest();
+Resource* ResourceFetcher::MatchPreload(const FetchParameters& params,
+                                        Resource::Type type) {
+  auto it = preloads_.find(PreloadKey(params.Url(), type));
+  if (it == preloads_.end())
+    return nullptr;
 
-  if (!existing_resource)
-    return kLoad;
+  Resource* resource = it->value;
 
-  // If the existing resource is loading and the associated fetcher is not equal
-  // to |this|, we must not use the resource. Otherwise, CSP violation may
-  // happen in redirect handling.
-  if (existing_resource->Loader() &&
-      existing_resource->Loader()->Fetcher() != this) {
-    return kReload;
+  RecordSriResourceIntegrityMismatchEvent(kCheckingForIntegrityMismatch);
+  if (resource->MustRefetchDueToIntegrityMetadata(params))
+    return nullptr;
+
+  if (params.IsSpeculativePreload())
+    return resource;
+  if (params.IsLinkPreload()) {
+    resource->SetLinkPreload(true);
+    return resource;
   }
 
-  // Checks if the resource has an explicit policy about integrity metadata.
-  //
-  // This is necessary because ScriptResource and CSSStyleSheetResource objects
-  // do not keep the raw data around after the source is accessed once, so if
-  // the resource is accessed from the MemoryCache for a second time, there is
-  // no way to redo an integrity check.
-  //
-  // Thus, Blink implements a scheme where it caches the integrity information
-  // for those resources after the first time it is checked, and if there is
-  // another request for that resource, with the same integrity metadata, Blink
-  // skips the integrity calculation. However, if the integrity metadata is a
-  // mismatch, the MemoryCache must be skipped here, and a new request for the
-  // resource must be made to get the raw data. This is expected to be an
-  // uncommon case, however, as it implies two same-origin requests to the same
-  // resource, but with different integrity metadata.
-  RecordSriResourceIntegrityMismatchEvent(kCheckingForIntegrityMismatch);
-  if (existing_resource->MustRefetchDueToIntegrityMetadata(fetch_params)) {
-    RecordSriResourceIntegrityMismatchEvent(kRefetchDueToIntegrityMismatch);
-    return kReload;
+  if (!IsReusableForPreloading(params, resource, false))
+    return nullptr;
+
+  resource->DecreasePreloadCount();
+  preloads_.erase(it);
+  matched_preloads_.push_back(resource);
+  return resource;
+}
+
+void ResourceFetcher::InsertAsPreloadIfNecessary(Resource* resource,
+                                                 const FetchParameters& params,
+                                                 Resource::Type type) {
+  if (!params.IsSpeculativePreload() && !params.IsLinkPreload())
+    return;
+  // CSP layout tests verify that preloads are subject to access checks by
+  // seeing if they are in the `preload started` list. Therefore do not add
+  // them to the list if the load is immediately denied.
+  if (resource->GetResourceError().IsAccessCheck())
+    return;
+  PreloadKey key(params.Url(), type);
+  if (preloads_.find(key) == preloads_.end()) {
+    preloads_.insert(key, resource);
+    resource->IncreasePreloadCount();
+    if (preloaded_urls_for_test_)
+      preloaded_urls_for_test_->insert(resource->Url().GetString());
   }
+}
 
-  // If the same URL has been loaded as a different type, we need to reload.
-  if (existing_resource->GetType() != type) {
-    // FIXME: If existingResource is a Preload and the new type is LinkPrefetch
-    // We really should discard the new prefetch since the preload has more
-    // specific type information! crbug.com/379893
-    // fast/dom/HTMLLinkElement/link-and-subresource-test hits this case.
-    RESOURCE_LOADING_DVLOG(1) << "ResourceFetcher::determineRevalidationPolicy "
-                                 "reloading due to type mismatch.";
-    return kReload;
-  }
-
-  // We already have a preload going for this URL.
-  if (fetch_params.IsSpeculativePreload() && existing_resource->IsPreloaded())
-    return kUse;
-
+bool ResourceFetcher::IsReusableForPreloading(const FetchParameters& params,
+                                              Resource* existing_resource,
+                                              bool is_static_data) const {
+  const ResourceRequest& request = params.GetResourceRequest();
   // Do not load from cache if images are not enabled. There are two general
   // cases:
   //
   // 1. Images are disabled. Don't ever load images, even if the image is cached
   // or it is a data: url. In this case, we "Reload" the image, then defer it
   // with resourceNeedsLoad() so that it never actually goes to the network.
   //
   // 2. Images are enabled, but not loaded automatically. In this case, we will
   // Use cached resources or data: urls, but will similarly fall back to a
   // deferred network load if we don't have the data available without a network
   // request. We check allowImage() here, which is affected by m_imagesEnabled
   // but not m_autoLoadImages, in order to allow for this differing behavior.
   //
   // TODO(japhet): Can we get rid of one of these settings?
   if (existing_resource->IsImage() &&
       !Context().AllowImage(images_enabled_, existing_resource->Url())) {
-    return kReload;
+    return false;
   }
 
   // Never use cache entries for downloadToFile / useStreamOnResponse requests.
   // The data will be delivered through other paths.
   if (request.DownloadToFile() || request.UseStreamOnResponse())
-    return kReload;
+    return false;
 
   // Never reuse opaque responses from a service worker for requests that are
   // not no-cors. https://crbug.com/625575
   if (existing_resource->GetResponse().WasFetchedViaServiceWorker() &&
       existing_resource->GetResponse().ServiceWorkerResponseType() ==
           kWebServiceWorkerResponseTypeOpaque &&
       request.GetFetchRequestMode() != WebURLRequest::kFetchRequestModeNoCORS) {
-    return kReload;
+    return false;
   }
 
-  if (!is_static_data && !existing_resource->CanReuse(fetch_params))
-    return kReload;
+  if (!is_static_data && !existing_resource->CanReuse(params))
+    return false;
 
   // Certain requests (e.g., XHRs) might have manually set headers that require
   // revalidation. In theory, this should be a Revalidate case. In practice, the
   // MemoryCache revalidation path assumes a whole bunch of things about how
   // revalidation works that manual headers violate, so punt to Reload instead.
   //
   // Similarly, a request with manually added revalidation headers can lead to a
   // 304 response for a request that wasn't flagged as a revalidation attempt.
   // Normally, successful revalidation will maintain the original response's
   // status code, but for a manual revalidation the response code remains 304.
   // In this case, the Resource likely has insufficient context to provide a
   // useful cache hit or revalidation. See http://crbug.com/643659
   if (!is_static_data &&
       (request.IsConditional() ||
        existing_resource->GetResponse().HttpStatusCode() == 304)) {
+    return false;
+  }
+
+  if (!is_static_data &&
+      !params.Options().CanReuseRequest(existing_resource->Options())) {
+    return false;
+  }
+
+  return true;
+}
+
+ResourceFetcher::RevalidationPolicy
+ResourceFetcher::DetermineRevalidationPolicy(
+    Resource::Type type,
+    const FetchParameters& fetch_params,
+    Resource* existing_resource,
+    bool is_static_data) const {
+  const ResourceRequest& request = fetch_params.GetResourceRequest();
+
+  if (!existing_resource)
+    return kLoad;
+
+  // If the existing resource is loading and the associated fetcher is not equal
+  // to |this|, we must not use the resource. Otherwise, CSP violation may
+  // happen in redirect handling.
+  if (existing_resource->Loader() &&
+      existing_resource->Loader()->Fetcher() != this) {
     return kReload;
   }
 
-  if (!is_static_data &&
-      !fetch_params.Options().CanReuseRequest(existing_resource->Options())) {
+  // Checks if the resource has an explicit policy about integrity metadata.
+  //
+  // This is necessary because ScriptResource and CSSStyleSheetResource objects
+  // do not keep the raw data around after the source is accessed once, so if
+  // the resource is accessed from the MemoryCache for a second time, there is
+  // no way to redo an integrity check.
+  //
+  // Thus, Blink implements a scheme where it caches the integrity information
+  // for those resources after the first time it is checked, and if there is
+  // another request for that resource, with the same integrity metadata, Blink
+  // skips the integrity calculation. However, if the integrity metadata is a
+  // mismatch, the MemoryCache must be skipped here, and a new request for the
+  // resource must be made to get the raw data. This is expected to be an
+  // uncommon case, however, as it implies two same-origin requests to the same
+  // resource, but with different integrity metadata.
+  RecordSriResourceIntegrityMismatchEvent(kCheckingForIntegrityMismatch);
+  if (existing_resource->MustRefetchDueToIntegrityMetadata(fetch_params)) {
+    RecordSriResourceIntegrityMismatchEvent(kRefetchDueToIntegrityMismatch);
     return kReload;
   }
 
-  // Always use preloads.
-  if (existing_resource->IsPreloaded())
-    return kUse;
+  // If the same URL has been loaded as a different type, we need to reload.
+  if (existing_resource->GetType() != type) {
+    // FIXME: If existingResource is a Preload and the new type is LinkPrefetch
+    // We really should discard the new prefetch since the preload has more
+    // specific type information! crbug.com/379893
+    // fast/dom/HTMLLinkElement/link-and-subresource-test hits this case.
+    RESOURCE_LOADING_DVLOG(1) << "ResourceFetcher::determineRevalidationPolicy "
+                                 "reloading due to type mismatch.";
+    return kReload;
+  }
+
+  // If |existing_resource| is not reusable as a preloaded resource, it should
+  // not be reusable as a normal resource as well.
+  if (!IsReusableForPreloading(fetch_params, existing_resource, is_static_data))

[Yoav Weiss, 2017/05/12 16:33:57]

Can we maybe change the name of `isReusableForPreloading`, as it's applied to to
all resources?

[kinuko, 2017/05/15 02:09:04]

+1, felt slightly confusing.

[yhirano, 2017/05/16 11:19:24]

I couldn't come up with a better name. Do you have any recommendation?

[kinuko, 2017/05/16 22:58:53]

IsReusableAlsoForPreloading ...?  (I'm not good at naming either)

[yhirano, 2017/05/17 04:24:44]

Done.

+    return kReload;
 
   // If resource was populated from a SubstituteData load or data: url, use it.
   if (is_static_data)
     return kUse;
 
   // Don't reload resources while pasting.
   if (allow_stale_resources_)
     return kUse;
 
   // WebCachePolicy::ReturnCacheDataElseLoad uses the cache no matter what.
@@ skipping 140 matching lines @@
 }
 
 int ResourceFetcher::BlockingRequestCount() const {
   return loaders_.size();
 }
 
 int ResourceFetcher::NonblockingRequestCount() const {
   return non_blocking_loaders_.size();
 }
 
-void ResourceFetcher::PreloadStarted(Resource* resource) {
-  if (preloads_.Contains(resource))
-    return;
-  resource->IncreasePreloadCount();
-
-  preloads_.insert(resource);
-
-  if (preloaded_urls_for_test_)
-    preloaded_urls_for_test_->insert(resource->Url().GetString());
-}
-
 void ResourceFetcher::EnableIsPreloadedForTest() {
   if (preloaded_urls_for_test_)
     return;
   preloaded_urls_for_test_ = WTF::WrapUnique(new HashSet<String>);
 
-  for (const auto& resource : preloads_)
+  for (const auto& pair : preloads_) {
+    Resource* resource = pair.value;
     preloaded_urls_for_test_->insert(resource->Url().GetString());
+  }
 }
 
 bool ResourceFetcher::IsPreloadedForTest(const KURL& url) const {
   DCHECK(preloaded_urls_for_test_);
   return preloaded_urls_for_test_->Contains(url.GetString());
 }
 
 void ResourceFetcher::ClearPreloads(ClearPreloadsPolicy policy) {
   LogPreloadStats(policy);
 
-  for (const auto& resource : preloads_) {
+  Vector<PreloadKey> keys_to_be_removed;
+  for (const auto& pair : preloads_) {
+    Resource* resource = pair.value;
     if (policy == kClearAllPreloads || !resource->IsLinkPreload()) {
       resource->DecreasePreloadCount();
       if (resource->GetPreloadResult() == Resource::kPreloadNotReferenced)
-        GetMemoryCache()->Remove(resource.Get());
-      preloads_.erase(resource);
+        GetMemoryCache()->Remove(resource);
+      keys_to_be_removed.push_back(pair.key);
     }
   }
+  preloads_.RemoveAll(keys_to_be_removed);
+
+  matched_preloads_.clear();
 }
 
 void ResourceFetcher::WarnUnusedPreloads() {
-  for (const auto& resource : preloads_) {
+  for (const auto& pair : preloads_) {
+    Resource* resource = pair.value;
     if (resource && resource->IsLinkPreload() &&
         resource->GetPreloadResult() == Resource::kPreloadNotReferenced) {
       Context().AddConsoleMessage(
           "The resource " + resource->Url().GetString() +
               " was preloaded using link preload but not used within a few "
               "seconds from the window's load event. Please make sure it "
               "wasn't preloaded for nothing.",
           FetchContext::kLogWarningMessage);
     }
   }
@@ skipping 86 matching lines @@
 
   resource_timing_info_map_.Take(resource);
 
   bool is_internal_request = resource->Options().initiator_info.name ==
                              FetchInitiatorTypeNames::internal;
 
   Context().DispatchDidFail(resource->Identifier(), error,
                             resource->GetResponse().EncodedDataLength(),
                             is_internal_request);
 
+  if (error.IsCancellation())
+    RemovePreload(resource);
   resource->FinishAsError(error);
 
   HandleLoadCompletion(resource);
 }
 
 void ResourceFetcher::MoveResourceLoaderToNonBlocking(ResourceLoader* loader) {
   DCHECK(loader);
   // TODO(yoav): Convert CHECK to DCHECK if no crash reports come in.
   CHECK(loaders_.Contains(loader));
   non_blocking_loaders_.insert(loader);
@@ skipping 138 matching lines @@
   unsigned fonts = 0;
   unsigned font_misses = 0;
   unsigned medias = 0;
   unsigned media_misses = 0;
   unsigned text_tracks = 0;
   unsigned text_track_misses = 0;
   unsigned imports = 0;
   unsigned import_misses = 0;
   unsigned raws = 0;
   unsigned raw_misses = 0;
-  for (const auto& resource : preloads_) {
+  for (const auto& pair : preloads_) {
+    Resource* resource = pair.value;
     // Do not double count link rel preloads. These do not get cleared if the
     // ClearPreloadsPolicy is only clearing speculative markup preloads.
     if (resource->IsLinkPreload() &&
         policy == kClearSpeculativeMarkupPreloads) {
       continue;
     }
     int miss_count =
         resource->GetPreloadResult() == Resource::kPreloadNotReferenced ? 1 : 0;
     switch (resource->GetType()) {
       case Resource::kImage:
@@ skipping 133 matching lines @@
   RequestLoadStarted(resource->Identifier(), resource, params, kUse);
 }
 
 DEFINE_TRACE(ResourceFetcher) {
   visitor->Trace(context_);
   visitor->Trace(archive_);
   visitor->Trace(loaders_);
   visitor->Trace(non_blocking_loaders_);
   visitor->Trace(document_resources_);
   visitor->Trace(preloads_);
+  visitor->Trace(matched_preloads_);
   visitor->Trace(resource_timing_info_map_);
 }
 
 }  // namespace blink