Adding WebContent-free Download

content/browser/download/url_downloader.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/download/url_downloader.h"
+
+#include "base/location.h"
+#include "base/thread_task_runner_handle.h"
+#include "content/browser/appcache/appcache_interceptor.h"
+#include "content/browser/download/download_resource_handler.h"
+#include "content/browser/loader/resource_request_info_impl.h"
+#include "content/browser/service_worker/service_worker_request_handler.h"
+#include "content/browser/ssl/ssl_policy.h"
+#include "content/common/ssl_status_serialization.h"
+#include "content/public/browser/cert_store.h"
+#include "content/public/browser/download_save_info.h"
+#include "content/public/browser/resource_context.h"
+#include "content/public/browser/signed_certificate_timestamp_store.h"
+#include "content/public/common/process_type.h"
+#include "content/public/common/resource_response.h"
+#include "content/public/common/security_style.h"
+#include "net/base/io_buffer.h"
+#include "net/base/load_flags.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_status_code.h"
+#include "ui/base/page_transition_types.h"
+
+namespace content {
+// The average private bytes increase of the browser for each new pending
+// request. Experimentally obtained.
+static const int kAvgBytesPerOutstandingRequest = 4400;
+
+int CalculateApproximateMemoryCost(net::URLRequest* request) {
+  // The following fields should be a minor size contribution (experimentally
+  // on the order of 100). However since they are variable length, it could
+  // in theory be a sizeable contribution.
+  int strings_cost = request->extra_request_headers().ToString().size() +
+                     request->original_url().spec().size() +
+                     request->referrer().size() + request->method().size();
+
+  // Note that this expression will typically be dominated by:
+  // |kAvgBytesPerOutstandingRequest|.
+  return kAvgBytesPerOutstandingRequest + strings_cost;
+}
+
+void StoreSignedCertificateTimestamps(
+    const net::SignedCertificateTimestampAndStatusList& sct_list,
+    int process_id,
+    SignedCertificateTimestampIDStatusList* sct_ids) {
+  SignedCertificateTimestampStore* sct_store(
+      SignedCertificateTimestampStore::GetInstance());
+
+  for (auto iter = sct_list.begin(); iter != sct_list.end(); ++iter) {
+    const int sct_id(sct_store->Store(iter->sct.get(), process_id));
+    sct_ids->push_back(
+        SignedCertificateTimestampIDAndStatus(sct_id, iter->status));
+  }
+}
+
+void GetSSLStatusForRequest(const GURL& url,
+                            const net::SSLInfo& ssl_info,
+                            int child_id,
+                            SSLStatus* ssl_status) {
+  DCHECK(ssl_info.cert);
+
+  int cert_id =
+      CertStore::GetInstance()->StoreCert(ssl_info.cert.get(), child_id);
+
+  SignedCertificateTimestampIDStatusList signed_certificate_timestamp_ids;
+  StoreSignedCertificateTimestamps(ssl_info.signed_certificate_timestamps,
+                                   child_id, &signed_certificate_timestamp_ids);
+
+  *ssl_status = SSLStatus(SSLPolicy::GetSecurityStyleForResource(
+                              url, cert_id, ssl_info.cert_status),
+                          cert_id, signed_certificate_timestamp_ids, ssl_info);
+}
+
+void PopulateResourceResponse(ResourceRequestInfoImpl* info,
+                              net::URLRequest* request,
+                              ResourceResponse* response) {
+  response->head.request_time = request->request_time();
+  response->head.response_time = request->response_time();
+  response->head.headers = request->response_headers();
+  request->GetCharset(&response->head.charset);
+  response->head.content_length = request->GetExpectedContentSize();
+  request->GetMimeType(&response->head.mime_type);
+  net::HttpResponseInfo response_info = request->response_info();
+  response->head.was_fetched_via_spdy = response_info.was_fetched_via_spdy;
+  response->head.was_npn_negotiated = response_info.was_npn_negotiated;
+  response->head.npn_negotiated_protocol =
+      response_info.npn_negotiated_protocol;
+  response->head.connection_info = response_info.connection_info;
+  response->head.was_fetched_via_proxy = request->was_fetched_via_proxy();
+  response->head.proxy_server = response_info.proxy_server;
+  response->head.socket_address = request->GetSocketAddress();
+  const content::ResourceRequestInfo* request_info =
+      content::ResourceRequestInfo::ForRequest(request);
+  if (request_info)
+    response->head.is_using_lofi = request_info->IsUsingLoFi();
+  if (ServiceWorkerRequestHandler* handler =
+          ServiceWorkerRequestHandler::GetHandler(request)) {
+    handler->GetExtraResponseInfo(&response->head);
+  }
+  AppCacheInterceptor::GetExtraResponseInfo(
+      request, &response->head.appcache_id,
+      &response->head.appcache_manifest_url);
+  if (info->is_load_timing_enabled())
+    request->GetLoadTimingInfo(&response->head.load_timing);
+
+  if (request->ssl_info().cert.get()) {
+    SSLStatus ssl_status;
+    GetSSLStatusForRequest(request->url(), request->ssl_info(),
+                           info->GetChildID(), &ssl_status);
+    response->head.security_info = SerializeSecurityInfo(ssl_status);
+  } else {
+    // We should not have any SSL state.
+    DCHECK(!request->ssl_info().cert_status);
+    DCHECK_EQ(request->ssl_info().security_bits, -1);
+    DCHECK_EQ(request->ssl_info().key_exchange_info, 0);
+    DCHECK(!request->ssl_info().connection_status);
+  }
+}
+
+DownloadInterruptReason UrlDownloader::BeginDownload(
+    base::WeakPtr<DownloadManagerImpl> download_manager,
+    scoped_ptr<net::URLRequest> request,
+    const Referrer& referrer,
+    bool is_content_initiated,
+    ResourceContext* context,
+    bool prefer_cache,
+    bool do_not_prompt_for_login,
+    scoped_ptr<DownloadSaveInfo> save_info,
+    uint32 download_id,
+    const DownloadUrlParameters::OnStartedCallback& started_callback) {
+  if (!referrer.url.is_valid())
+    request->SetReferrer(std::string());
+  else
+    request->SetReferrer(referrer.url.spec());
+
+  int extra_load_flags = net::LOAD_NORMAL;
+  if (prefer_cache) {
+    // If there is upload data attached, only retrieve from cache because there
+    // is no current mechanism to prompt the user for their consent for a
+    // re-post. For GETs, try to retrieve data from the cache and skip
+    // validating the entry if present.
+    if (request->get_upload() != NULL)
+      extra_load_flags |= net::LOAD_ONLY_FROM_CACHE;
+    else
+      extra_load_flags |= net::LOAD_PREFERRING_CACHE;
+  } else {
+    extra_load_flags |= net::LOAD_DISABLE_CACHE;
+  }
+  request->SetLoadFlags(request->load_flags() | extra_load_flags);
+
+  // We treat a download as a main frame load, and thus update the policy URL on
+  // redirects.
+  //
+  // TODO(davidben): Is this correct? If this came from a
+  // ViewHostMsg_DownloadUrl in a frame, should it have first-party URL set
+  // appropriately?
+  request->set_first_party_url_policy(
+      net::URLRequest::UPDATE_FIRST_PARTY_URL_ON_REDIRECT);
+
+  ResourceRequestInfoImpl* extra_info = new ResourceRequestInfoImpl(

[asanka, 2015/11/25 15:53:57]

I'm still not too happy about creating this ResourceRequestInfoImpl here. I know
we are putting a stop-gap in place, but it still feels like we should pull out
the bits in DownloadResourceHandler that deal with request driving and response
handling and invoke it directly rather than try to wrap the DRH itself.

Have you checked with davidben about what he thinks about this?

That said, I'm willing to yield to //content owner opinion on this.

[svaldez, 2015/11/25 17:53:28]

Wouldn't we still need to create the RRII here anyway, since most of the stack
later on assumes that we've got an RRII associated with the URLRequest.
Otherwise we'd need to pipe this through to the DownloadManager separately.
Also, after removing the request driving/response handling from
DownloadResourceHandler, wouldn't DRH be reduced to just creating the
DownloadCreateInfo from RRII, and passing that along to the new request driver?

[asanka, 2015/11/25 19:43:11]

If we are using a captive DRH, we'd need an RRII. But the suggestion is to avoid
doing that. All the information we need to establish the new download can be
extracted from the URLRequest. See
https://codereview.chromium.org/23496076/diff/1/content/browser/download/down...
which extracts the core of DRH so that we can use it from both URLDownloader and
DRH.

+      PROCESS_TYPE_RENDERER, -1, -1,
+      -1,  // frame_tree_node_id
+      0, -1, -1,
+      false,  // is_main_frame
+      false,  // parent_is_main_frame
+      -1,     // parent_render_frame_id
+      RESOURCE_TYPE_SUB_RESOURCE, ui::PAGE_TRANSITION_LINK,
+      false,  // should_replace_current_entry
+      true,   // is_download
+      false,  // is_stream
+      true,   // allow_download
+      false,  // has_user_gesture
+      false,  // enable_load_timing
+      false,  // enable_upload_progress
+      false,  // do_not_prompt_for_login
+      blink::WebReferrerPolicyDefault, blink::WebPageVisibilityStateVisible,
+      context,
+      base::WeakPtr<ResourceMessageFilter>(),  // filter
+      false,                                   // report_raw_headers
+      true,                                    // is_async
+      false);                                  // is_using_lofi
+
+  extra_info->set_do_not_prompt_for_login(do_not_prompt_for_login);
+  extra_info->AssociateWithRequest(request.get());  // Request takes ownership.
+
+  if (request->url().SchemeIs(url::kBlobScheme))
+    return DOWNLOAD_INTERRUPT_REASON_FILE_FAILED;
+
+  extra_info->set_memory_cost(CalculateApproximateMemoryCost(request.get()));
+
+  DownloadResourceHandler* handler = new DownloadResourceHandler(
+      download_id, request.get(), started_callback, save_info.Pass());
+  handler->set_download_manager(download_manager);
+
+  // From this point forward, the |UrlDownloader| is responsible for
+  // |started_callback|.
+  linked_ptr<UrlDownloader> downloader(
+      new UrlDownloader(request.Pass(), scoped_ptr<ResourceHandler>(handler)));
+
+  downloader->Start();
+
+  g_active_downloaders.push_back(downloader);
+
+  return DOWNLOAD_INTERRUPT_REASON_NONE;
+}
+
+UrlDownloader::UrlDownloader(scoped_ptr<net::URLRequest> request,
+                             scoped_ptr<ResourceHandler> handler)
+    : request_(request.Pass()),
+      handler_(handler.Pass()),
+      weak_ptr_factory_(this) {}
+
+UrlDownloader::~UrlDownloader() {
+  handler_.reset();
+}
+
+void UrlDownloader::Start() {
+  // Give the handler a chance to delay the URLRequest from being started.
+  bool defer_start = false;
+  if (!handler_->OnWillStart(request_->url(), &defer_start)) {
+    request_->CancelWithError(net::ERR_ABORTED);
+    return;
+  }
+
+  DCHECK(!defer_start);
+  DCHECK(!request_->is_pending());
+
+  if (!request_->status().is_success()) {
+    return;
+  }
+
+  request_->set_delegate(this);
+  request_->Start();
+}
+
+void UrlDownloader::OnReceivedRedirect(net::URLRequest* request,
+                                       const net::RedirectInfo& redirect_info,
+                                       bool* defer_redirect) {
+  DVLOG(1) << "OnReceivedRedirect: " << request_->url().spec();
+  request_->CancelWithError(net::ERR_ABORTED);
+}
+
+void UrlDownloader::OnResponseStarted(net::URLRequest* request) {
+  DVLOG(1) << "OnResponseStarted: " << request_->url().spec();
+
+  if (!request_->status().is_success()) {
+    ResponseCompleted();
+    return;
+  }
+
+  ResourceRequestInfoImpl* info =
+      ResourceRequestInfoImpl::ForRequest(request_.get());
+  scoped_refptr<ResourceResponse> response(new ResourceResponse());
+  PopulateResourceResponse(info, request_.get(), response.get());
+
+  bool defer = false;
+  if (!handler_->OnResponseStarted(response.get(), &defer)) {
+    request_->CancelWithError(net::ERR_ABORTED);
+    return;
+  }
+  DCHECK(!defer);
+
+  if (request_->status().is_success())
+    StartReading(false);  // Read the first chunk.
+  else
+    ResponseCompleted();
+}
+
+void UrlDownloader::OnReadCompleted(net::URLRequest* request, int bytes_read) {
+  DVLOG(1) << "OnReadCompleted: \"" << request_->url().spec() << "\""
+           << " bytes_read = " << bytes_read;
+
+  // bytes_read == -1 always implies an error.
+  if (bytes_read == -1 || !request_->status().is_success()) {
+    ResponseCompleted();
+    return;
+  }
+
+  DCHECK(bytes_read >= 0);
+  DCHECK(request_->status().is_success());
+
+  bool defer = false;
+  if (!handler_->OnReadCompleted(bytes_read, &defer)) {
+    request_->CancelWithError(net::ERR_ABORTED);
+    return;
+  }
+  DCHECK(!defer);
+
+  if (!request_->status().is_success())
+    return;
+
+  if (bytes_read > 0) {
+    StartReading(true);  // Read the next chunk.
+  } else {
+    // URLRequest reported an EOF. Call ResponseCompleted.
+    DCHECK_EQ(0, bytes_read);
+    ResponseCompleted();
+  }
+}
+
+void UrlDownloader::StartReading(bool is_continuation) {
+  int bytes_read;
+
+  // Make sure we track the buffer in at least one place.  This ensures it gets
+  // deleted even in the case the request has already finished its job and
+  // doesn't use the buffer.
+  scoped_refptr<net::IOBuffer> buf;
+  int buf_size;
+  if (!handler_->OnWillRead(&buf, &buf_size, -1)) {
+    request_->CancelWithError(net::ERR_ABORTED);
+    return;
+  }
+
+  DCHECK(buf.get());
+  DCHECK(buf_size > 0);
+
+  request_->Read(buf.get(), buf_size, &bytes_read);
+
+  // If IO is pending, wait for the URLRequest to call OnReadCompleted.
+  if (request_->status().is_io_pending())
+    return;
+
+  if (!is_continuation || bytes_read <= 0) {
+    OnReadCompleted(request_.get(), bytes_read);
+  } else {
+    // Else, trigger OnReadCompleted asynchronously to avoid starving the IO
+    // thread in case the URLRequest can provide data synchronously.
+    base::ThreadTaskRunnerHandle::Get()->PostTask(
+        FROM_HERE,
+        base::Bind(&UrlDownloader::OnReadCompleted,
+                   weak_ptr_factory_.GetWeakPtr(), request_.get(), bytes_read));
+  }
+}
+
+void UrlDownloader::ResponseCompleted() {
+  DVLOG(1) << "ResponseCompleted: " << request_->url().spec();
+
+  ResourceRequestInfoImpl* info =
+      ResourceRequestInfoImpl::ForRequest(request_.get());
+
+  std::string security_info;
+  const net::SSLInfo& ssl_info = request_->ssl_info();
+  if (ssl_info.cert.get() != NULL) {
+    SSLStatus ssl_status;
+    GetSSLStatusForRequest(request_->url(), ssl_info, info->GetChildID(),
+                           &ssl_status);
+
+    security_info = SerializeSecurityInfo(ssl_status);
+  }
+
+  bool defer = false;
+  handler_->OnResponseCompleted(request_->status(), security_info, &defer);
+  DCHECK(!defer);
+}
+
+}  // namespace content