PlzNavigate: Add first version of NavigationURLLoader.

content/browser/loader/resource_dispatcher_host_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // See http://dev.chromium.org/developers/design-documents/multi-process-resource-loading
 
 #include "content/browser/loader/resource_dispatcher_host_impl.h"
 
 #include <algorithm>
 #include <set>
@@ skipping 19 matching lines @@
 #include "content/browser/download/download_resource_handler.h"
 #include "content/browser/download/save_file_manager.h"
 #include "content/browser/download/save_file_resource_handler.h"
 #include "content/browser/fileapi/chrome_blob_storage_context.h"
 #include "content/browser/frame_host/navigation_request_info.h"
 #include "content/browser/frame_host/navigator.h"
 #include "content/browser/loader/async_resource_handler.h"
 #include "content/browser/loader/buffered_resource_handler.h"
 #include "content/browser/loader/cross_site_resource_handler.h"
 #include "content/browser/loader/detachable_resource_handler.h"
+#include "content/browser/loader/navigation_resource_handler.h"
+#include "content/browser/loader/navigation_url_loader_impl_core.h"
 #include "content/browser/loader/power_save_block_resource_throttle.h"
 #include "content/browser/loader/redirect_to_file_resource_handler.h"
 #include "content/browser/loader/resource_message_filter.h"
 #include "content/browser/loader/resource_request_info_impl.h"
 #include "content/browser/loader/stream_resource_handler.h"
 #include "content/browser/loader/sync_resource_handler.h"
 #include "content/browser/loader/throttling_resource_handler.h"
 #include "content/browser/loader/upload_data_stream_builder.h"
 #include "content/browser/renderer_host/render_view_host_delegate.h"
 #include "content/browser/renderer_host/render_view_host_impl.h"
@@ skipping 303 matching lines @@
     return;
   web_contents->DidGetResourceResponseStart(*details.get());
 }
 
 bool IsValidatedSCT(
     const net::SignedCertificateTimestampAndStatus& sct_status) {
   return sct_status.status == net::ct::SCT_STATUS_OK;
 }
 
 storage::BlobStorageContext* GetBlobStorageContext(
-    ResourceMessageFilter* filter) {
-  if (!filter->blob_storage_context())
+    ChromeBlobStorageContext* blob_storage_context) {
+  if (!blob_storage_context)
     return NULL;
-  return filter->blob_storage_context()->context();
+  return blob_storage_context->context();
 }
 
 void AttachRequestBodyBlobDataHandles(
     ResourceRequestBody* body,
     storage::BlobStorageContext* blob_context) {
   DCHECK(blob_context);
   for (size_t i = 0; i < body->elements()->size(); ++i) {
     const ResourceRequestBody::Element& element = (*body->elements())[i];
     if (element.type() != ResourceRequestBody::Element::TYPE_BLOB)
       continue;
@@ skipping 158 matching lines @@
       delete loaders;
     } else {
       ++i;
     }
   }
 
 #ifndef NDEBUG
   for (LoaderList::iterator i = loaders_to_cancel.begin();
        i != loaders_to_cancel.end(); ++i) {
     // There is no strict requirement that this be the case, but currently
-    // downloads, streams, detachable requests, and transferred requests are the
-    // only requests that aren't cancelled when the associated processes go
-    // away. It may be OK for this invariant to change in the future, but if
-    // this assertion fires without the invariant changing, then it's indicative
-    // of a leak.
+    // downloads, streams, detachable requests, transferred requests, and
+    // browser-owned requests are the only requests that aren't cancelled when
+    // the associated processes go away. It may be OK for this invariant to
+    // change in the future, but if this assertion fires without the invariant
+    // changing, then it's indicative of a leak.
     DCHECK((*i)->GetRequestInfo()->IsDownload() ||
            (*i)->GetRequestInfo()->is_stream() ||
            ((*i)->GetRequestInfo()->detachable_handler() &&
             (*i)->GetRequestInfo()->detachable_handler()->is_detached()) ||
+           (*i)->GetRequestInfo()->GetProcessType() == PROCESS_TYPE_BROWSER ||
            (*i)->is_transferring());
   }
 #endif
 
   loaders_to_cancel.clear();
 
   // Validate that no more requests for this context were added.
   for (LoaderMap::const_iterator i = pending_loaders_.begin();
        i != pending_loaders_.end(); ++i) {
     // http://crbug.com/90971
@@ skipping 603 matching lines @@
   const Referrer referrer(request_data.referrer, request_data.referrer_policy);
   SetReferrerForRequest(new_request.get(), referrer);
 
   net::HttpRequestHeaders headers;
   headers.AddHeadersFromString(request_data.headers);
   new_request->SetExtraRequestHeaders(headers);
 
   new_request->SetLoadFlags(load_flags);
 
   storage::BlobStorageContext* blob_context =
-      GetBlobStorageContext(filter_);
+      GetBlobStorageContext(filter_->blob_storage_context());
   // Resolve elements from request_body and prepare upload data.
   if (request_data.request_body.get()) {
     // |blob_context| could be null when the request is from the plugins because
     // ResourceMessageFilters created in PluginProcessHost don't have the blob
     // context.
     if (blob_context) {
       // Attaches the BlobDataHandles to request_body not to free the blobs and
       // any attached shareable files until upload completion. These data will
       // be used in UploadDataStream and ServiceWorkerURLRequestJob.
       AttachRequestBodyBlobDataHandles(
@@ skipping 111 matching lines @@
   }
 
   // Prefetches and <a ping> requests outlive their child process.
   if (!sync_result && IsDetachableResourceType(request_data.resource_type)) {
     handler.reset(new DetachableResourceHandler(
         request,
         base::TimeDelta::FromMilliseconds(kDefaultDetachableCancelDelayMs),
         handler.Pass()));
   }
 
-  // Install a CrossSiteResourceHandler for all main frame requests.  This will
-  // let us check whether a transfer is required and pause for the unload
-  // handler either if so or if a cross-process navigation is already under way.
-  bool is_swappable_navigation =
-      request_data.resource_type == RESOURCE_TYPE_MAIN_FRAME;
-  // If we are using --site-per-process, install it for subframes as well.
-  if (!is_swappable_navigation &&
-      base::CommandLine::ForCurrentProcess()->HasSwitch(
-          switches::kSitePerProcess)) {
-    is_swappable_navigation =
-        request_data.resource_type == RESOURCE_TYPE_SUB_FRAME;
+  // PlzNavigate: If using --enable-browser-side-navigation, the
+  // CrossSiteResourceHandler is not needed. This codepath is not used for the
+  // actual navigation request, but only the subsequent blob URL load. This does
+  // not require request transfers.

[mmenke, 2014/10/23 20:43:40]

What about renderer-initiated cross-site navigations, either when we expect them
from the start to be cross-site, or when we only learn they are after redirects?

[davidben, 2014/10/27 20:46:05]

So, that's part of the nice thing about inverting the control. We no longer care
whether a navigation becomes cross-site initially or after a redirect. We make a
request for the URL with no reference[1] to the renderer currently in the frame,
if there even is one. At OnResponseStarted, we pop over the UI thread which does
a cross-process vs same-process check once and spin up[2] a renderer from there.

The request transfer craziness is because we're not in a position where we can
do that today and have to undo our early renderer binding.

[1] Except that we ping it so it can trigger beforeunload and also see if this
is a same-document navigation; #fragment change and such.

[2] Well, when the navigation starts, we'd like to predictively spin up a
renderer based on the starting URL and throw it away if it turns out to have
redirected, so they actually can happen in parallel. But from the point of view
of the rest of the system, the renderer is bound late.

+  if (!base::CommandLine::ForCurrentProcess()->HasSwitch(
+          switches::kEnableBrowserSideNavigation)) {
+    // Install a CrossSiteResourceHandler for all main frame requests. This will
+    // check whether a transfer is required and, if so, pause for the UI thread
+    // to drive the transfer.
+    bool is_swappable_navigation =
+        request_data.resource_type == RESOURCE_TYPE_MAIN_FRAME;
+    // If we are using --site-per-process, install it for subframes as well.
+    if (!is_swappable_navigation &&
+        base::CommandLine::ForCurrentProcess()->HasSwitch(
+            switches::kSitePerProcess)) {
+      is_swappable_navigation =
+          request_data.resource_type == RESOURCE_TYPE_SUB_FRAME;
+    }
+    if (is_swappable_navigation && process_type == PROCESS_TYPE_RENDERER)
+      handler.reset(new CrossSiteResourceHandler(handler.Pass(), request));
   }
-  if (is_swappable_navigation && process_type == PROCESS_TYPE_RENDERER)
-    handler.reset(new CrossSiteResourceHandler(handler.Pass(), request));
 
+  return AddStandardHandlers(request, request_data.resource_type,
+                             resource_context, filter_->appcache_service(),
+                             child_id, route_id, handler.Pass());
+}
+
+scoped_ptr<ResourceHandler> ResourceDispatcherHostImpl::AddStandardHandlers(
+      net::URLRequest* request,

[nasko, 2014/10/23 22:36:17]

nit: wrong indent, need only 4 spaces, not 6.

[davidben, 2014/10/27 20:46:05]

Done.

+      ResourceType resource_type,
+      ResourceContext* resource_context,
+      AppCacheService* appcache_service,
+      int child_id,
+      int route_id,
+      scoped_ptr<ResourceHandler> handler) {
   // Insert a buffered event handler before the actual one.
   handler.reset(
       new BufferedResourceHandler(handler.Pass(), this, request));
 
   ScopedVector<ResourceThrottle> throttles;
   if (delegate_) {
     delegate_->RequestBeginning(request,
                                 resource_context,
-                                filter_->appcache_service(),
-                                request_data.resource_type,
+                                appcache_service,
+                                resource_type,
                                 &throttles);
   }
 
   if (request->has_upload()) {
     // Block power save while uploading data.
     throttles.push_back(new PowerSaveBlockResourceThrottle());
   }
 
   throttles.push_back(
       scheduler_->ScheduleRequest(child_id, route_id, request).release());
@@ skipping 426 matching lines @@
 
   return true;
 }
 
 void ResourceDispatcherHostImpl::FinishedWithResourcesForRequest(
     net::URLRequest* request) {
   ResourceRequestInfoImpl* info = ResourceRequestInfoImpl::ForRequest(request);
   IncrementOutstandingRequestsCount(-1, info);
 }
 
-void ResourceDispatcherHostImpl::StartNavigationRequest(
+void ResourceDispatcherHostImpl::BeginNavigationRequest(
+    ResourceContext* resource_context,
+    int64 frame_tree_node_id,
     const CommonNavigationParams& params,
     const NavigationRequestInfo& info,
     scoped_refptr<ResourceRequestBody> request_body,
-    int64 navigation_request_id,
-    int64 frame_node_id) {
-  NOTIMPLEMENTED();
-}
+    NavigationURLLoaderImplCore* loader) {
+  // PlzNavigate: BeginNavigationRequest currently should only be used for the
+  // browser-side navigations project.
+  CHECK(base::CommandLine::ForCurrentProcess()->HasSwitch(
+      switches::kEnableBrowserSideNavigation));
 
-void ResourceDispatcherHostImpl::CancelNavigationRequest(
-    int64 navigation_request_id,
-    int64 frame_node_id) {
-  NOTIMPLEMENTED();
+  ResourceType resource_type = info.is_main_frame ?
+      RESOURCE_TYPE_MAIN_FRAME : RESOURCE_TYPE_SUB_FRAME;
+
+  if (is_shutdown_ ||
+      // TODO(davidben): Check ShouldServiceRequest here. This is important; it
+      // needs to be checked relative to the child that /requested/ the
+      // navigation. It's where file upload checks, etc., come in.
+      (delegate_ && !delegate_->ShouldBeginRequest(
+          info.navigation_params.method,
+          params.url,
+          resource_type,
+          resource_context))) {
+    loader->NotifyRequestFailed(net::ERR_ABORTED);
+    return;
+  }
+
+  // Save the URL on the stack to help catch URLRequests which outlive their
+  // URLRequestContexts. See https://crbug.com/90971
+  char url_buf[128];
+  base::strlcpy(url_buf, params.url.spec().c_str(), arraysize(url_buf));
+  base::debug::Alias(url_buf);
+  CHECK(ContainsKey(active_resource_contexts_, resource_context));

[mmenke, 2014/10/23 20:43:40]

Is this needed?

[davidben, 2014/10/27 20:46:05]

I pulled that from the other entry points. I guess it's to catch if you're
passing in an invalid ResourceContext somehow? *shrug* Can also remove it if we
don't have any reason to believe it'll help with anything.

+
+  const net::URLRequestContext* request_context =
+      resource_context->GetRequestContext();
+
+  int load_flags = info.navigation_params.load_flags;
+  load_flags |= net::LOAD_VERIFY_EV_CERT;
+  if (info.is_main_frame) {
+    load_flags |= net::LOAD_MAIN_FRAME;
+  } else {
+    load_flags |= net::LOAD_SUB_FRAME;
+  }
+  // Add a flag to selectively bypass the data reduction proxy if the resource
+  // type is not an image.
+  load_flags |= net::LOAD_BYPASS_DATA_REDUCTION_PROXY;
+
+  // TODO(davidben): BuildLoadFlagsForRequest includes logic for
+  // CanSendCookiesForOrigin and CanReadRawCookies. Is this needed here?
+
+  // Sync loads should have maximum priority and should be the only
+  // requests that have the ignore limits flag set.
+  DCHECK_EQ(load_flags & net::LOAD_IGNORE_LIMITS, 0);

[mmenke, 2014/10/23 20:43:40]

DCHECK(!(load_flags & net::LOAD_IGNORE_LIMITS))?

[davidben, 2014/10/27 20:46:04]

Done.

+
+  // TODO(davidben): OverrideCookieStoreForRenderProcess handling for
+  // prerender. There may not be a renderer process yet, so we need to use the
+  // ResourceContext or something.
+  scoped_ptr<net::URLRequest> new_request;
+  new_request = request_context->CreateRequest(params.url, net::HIGHEST,
+                                               nullptr, nullptr);
+
+  new_request->set_method(info.navigation_params.method);
+  new_request->set_first_party_for_cookies(
+      info.first_party_for_cookies);
+  if (info.is_main_frame) {
+    new_request->set_first_party_url_policy(
+        net::URLRequest::UPDATE_FIRST_PARTY_URL_ON_REDIRECT);
+  }
+
+  SetReferrerForRequest(new_request.get(), params.referrer);
+
+  net::HttpRequestHeaders headers;
+  headers.AddHeadersFromString(info.navigation_params.headers);
+  new_request->SetExtraRequestHeaders(headers);
+
+  new_request->SetLoadFlags(load_flags);
+
+  // Resolve elements from request_body and prepare upload data.
+  if (info.navigation_params.request_body.get()) {
+    storage::BlobStorageContext* blob_context = GetBlobStorageContext(
+        GetChromeBlobStorageContextForResourceContext(resource_context));
+    AttachRequestBodyBlobDataHandles(
+        info.navigation_params.request_body.get(),
+        blob_context);
+    // TODO(davidben): The FileSystemContext is null here. In the case where
+    // another renderer requested this navigation, this should be the same
+    // FileSystemContext passed into ShouldServiceRequest.
+    new_request->set_upload(UploadDataStreamBuilder::Build(
+        info.navigation_params.request_body.get(),
+        blob_context,
+        nullptr,  // file_system_context
+        BrowserThread::GetMessageLoopProxyForThread(BrowserThread::FILE)
+            .get()));
+  }
+
+  request_id_--;
+
+  // Make extra info and read footer (contains request ID).
+  //
+  // TODO(davidben): Associate the request with the FrameTreeNode and/or tab so
+  // that IO thread -> UI thread hops will work.
+  ResourceRequestInfoImpl* extra_info =
+      new ResourceRequestInfoImpl(
+          PROCESS_TYPE_BROWSER,
+          -1,  // child_id
+          -1,  // route_id
+          -1,  // request_data.origin_pid,
+          request_id_,
+          -1,  // request_data.render_frame_id,
+          info.is_main_frame,
+          info.parent_is_main_frame,
+          -1,  // request_data.parent_render_frame_id,
+          resource_type,
+          params.transition,
+          // should_replace_current_entry. This was only maintained at layer for
+          // request transfers and isn't needed for browser-side navigations.

[mmenke, 2014/10/23 20:43:40]

So this will be needed when we handle renderer-initiated "navigations"
(Including reloads)

[davidben, 2014/10/27 20:46:05]

This is needed in the old navigation flow just for request transfers. Outside
request transfers, the RDH stack doesn't care about this value. Once PlzNavigate
is done, it's one of many many fields we can remove from ResourceRequestInfo.
:-)

(The reason for it is that request transfers require the resource loader stack
to be able to recreate any navigation. So all navigation state needs to be
plumbed into the resource layer.)



It's really unfortunate that browser-side vs renderer-side and browser-initiated
vs renderer-initiated are such similar terminology.

browser-side = PlzNavigate = browser drives navigation request.
benderer-side = current logic = renderer drives navigation request.

browser-initiated = navigation came from browser. Omnibox or whatever.
renderer-initiated = navigation came from a renderer. Click a link or whatever.

+          false,
+          false,  // is download
+          false,  // is stream
+          params.allow_download,
+          info.navigation_params.has_user_gesture,
+          true,
+          params.referrer.policy,
+          // TODO(davidben): This is only used for prerenders. Replace
+          // is_showing with something for that. Or maybe it just comes from the
+          // same mechanism as the cookie one.
+          blink::WebPageVisibilityStateVisible,
+          resource_context,
+          base::WeakPtr<ResourceMessageFilter>(),  // filter
+          true);
+  // Request takes ownership.
+  extra_info->AssociateWithRequest(new_request.get());
+
+  if (new_request->url().SchemeIs(url::kBlobScheme)) {
+    // Hang on to a reference to ensure the blob is not released prior
+    // to the job being started.
+    ChromeBlobStorageContext* blob_context =
+        GetChromeBlobStorageContextForResourceContext(resource_context);
+    storage::BlobProtocolHandler::SetRequestedBlobDataHandle(
+        new_request.get(),
+        blob_context->context()->GetBlobDataFromPublicURL(new_request->url()));
+  }
+
+  // TODO(davidben): Attach ServiceWorkerRequestHandler.
+
+  // TODO(davidben): Attach AppCacheInterceptor.
+
+  scoped_ptr<ResourceHandler> handler(new NavigationResourceHandler(
+      new_request.get(), loader));
+
+  handler = AddStandardHandlers(new_request.get(), resource_type,
+                                resource_context,
+                                nullptr,  // TODO(davidben): appcache_service
+                                -1,  // TODO(davidben): child_id
+                                -1,  // TODO(davidben): route_id

[mmenke, 2014/10/23 20:43:40]

Looks like these are only used by the scheduler - may be able to get away
without them, or could use the ones for the old renderer, and it probably
wouldn't be the end of the world.  Just be beginning of the end of the world.

[davidben, 2014/10/27 20:46:05]

Yeah, it's currently a no-op because it can't find the entry in the ClientMap.
I've revised the TODO.

+                                handler.Pass());
+
+  BeginRequestInternal(new_request.Pass(), handler.Pass());
 }
 
 // static
 int ResourceDispatcherHostImpl::CalculateApproximateMemoryCost(
     net::URLRequest* request) {
   // The following fields should be a minor size contribution (experimentally
   // on the order of 100). However since they are variable length, it could
   // in theory be a sizeable contribution.
   int strings_cost = request->extra_request_headers().ToString().size() +
                      request->original_url().spec().size() +
@@ skipping 363 matching lines @@
 
   // Add a flag to selectively bypass the data reduction proxy if the resource
   // type is not an image.
   if (request_data.resource_type != RESOURCE_TYPE_IMAGE)
     load_flags |= net::LOAD_BYPASS_DATA_REDUCTION_PROXY;
 
   return load_flags;
 }
 
 }  // namespace content