Adding WebContent-free Download

content/browser/download/url_downloader.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/download/url_downloader.h"
+
+#include "base/location.h"
+#include "base/thread_task_runner_handle.h"
+#include "content/browser/appcache/appcache_interceptor.h"
+#include "content/browser/download/download_resource_handler.h"
+#include "content/browser/loader/resource_request_info_impl.h"
+#include "content/browser/service_worker/service_worker_request_handler.h"
+#include "content/browser/ssl/ssl_policy.h"
+#include "content/common/ssl_status_serialization.h"
+#include "content/public/browser/cert_store.h"
+#include "content/public/browser/download_save_info.h"
+#include "content/public/browser/resource_context.h"
+#include "content/public/browser/signed_certificate_timestamp_store.h"
+#include "content/public/common/process_type.h"
+#include "content/public/common/resource_response.h"
+#include "content/public/common/security_style.h"
+#include "net/base/io_buffer.h"
+#include "net/base/load_flags.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_status_code.h"
+#include "ui/base/page_transition_types.h"
+
+namespace content {
+void SetReferrerForRequest(net::URLRequest* request, const Referrer& referrer) {

[asanka, 2015/11/20 19:57:46]

For the URL downloader case, since the request is not originating from a
document, it also doesn't have a source for referrer policy. We can require that
the any required policies already be applied to the referrer_url and use it
as-is.

[svaldez, 2015/11/23 15:18:57]

Done.

+  if (!referrer.url.is_valid())
+    request->SetReferrer(std::string());
+  else
+    request->SetReferrer(referrer.url.spec());
+
+  net::URLRequest::ReferrerPolicy net_referrer_policy =
+      net::URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE;
+  switch (referrer.policy) {
+    case blink::WebReferrerPolicyAlways:
+    case blink::WebReferrerPolicyNever:
+    case blink::WebReferrerPolicyOrigin:
+      net_referrer_policy = net::URLRequest::NEVER_CLEAR_REFERRER;
+      break;
+    case blink::WebReferrerPolicyNoReferrerWhenDowngrade:
+      net_referrer_policy =
+          net::URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE;
+      break;
+    case blink::WebReferrerPolicyOriginWhenCrossOrigin:
+      net_referrer_policy =
+          net::URLRequest::ORIGIN_ONLY_ON_TRANSITION_CROSS_ORIGIN;
+      break;
+    case blink::WebReferrerPolicyDefault:
+    default:
+      net_referrer_policy =
+          net::URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE;
+      break;
+  }
+  request->set_referrer_policy(net_referrer_policy);
+}
+
+// The average private bytes increase of the browser for each new pending
+// request. Experimentally obtained.
+static const int kAvgBytesPerOutstandingRequest = 4400;
+
+int CalculateApproximateMemoryCost(net::URLRequest* request) {
+  // The following fields should be a minor size contribution (experimentally
+  // on the order of 100). However since they are variable length, it could
+  // in theory be a sizeable contribution.
+  int strings_cost = request->extra_request_headers().ToString().size() +
+                     request->original_url().spec().size() +
+                     request->referrer().size() + request->method().size();
+
+  // Note that this expression will typically be dominated by:
+  // |kAvgBytesPerOutstandingRequest|.
+  return kAvgBytesPerOutstandingRequest + strings_cost;
+}
+
+void StoreSignedCertificateTimestamps(
+    const net::SignedCertificateTimestampAndStatusList& sct_list,
+    int process_id,
+    SignedCertificateTimestampIDStatusList* sct_ids) {
+  SignedCertificateTimestampStore* sct_store(
+      SignedCertificateTimestampStore::GetInstance());
+
+  for (auto iter = sct_list.begin(); iter != sct_list.end(); ++iter) {
+    const int sct_id(sct_store->Store(iter->sct.get(), process_id));
+    sct_ids->push_back(
+        SignedCertificateTimestampIDAndStatus(sct_id, iter->status));
+  }
+}
+
+void GetSSLStatusForRequest(const GURL& url,
+                            const net::SSLInfo& ssl_info,
+                            int child_id,
+                            SSLStatus* ssl_status) {
+  DCHECK(ssl_info.cert);
+
+  int cert_id =
+      CertStore::GetInstance()->StoreCert(ssl_info.cert.get(), child_id);
+
+  SignedCertificateTimestampIDStatusList signed_certificate_timestamp_ids;
+  StoreSignedCertificateTimestamps(ssl_info.signed_certificate_timestamps,
+                                   child_id, &signed_certificate_timestamp_ids);
+
+  *ssl_status = SSLStatus(SSLPolicy::GetSecurityStyleForResource(
+                              url, cert_id, ssl_info.cert_status),
+                          cert_id, signed_certificate_timestamp_ids, ssl_info);
+}
+
+void PopulateResourceResponse(ResourceRequestInfoImpl* info,
+                              net::URLRequest* request,
+                              ResourceResponse* response) {
+  response->head.request_time = request->request_time();
+  response->head.response_time = request->response_time();
+  response->head.headers = request->response_headers();
+  request->GetCharset(&response->head.charset);
+  response->head.content_length = request->GetExpectedContentSize();
+  request->GetMimeType(&response->head.mime_type);
+  net::HttpResponseInfo response_info = request->response_info();
+  response->head.was_fetched_via_spdy = response_info.was_fetched_via_spdy;
+  response->head.was_npn_negotiated = response_info.was_npn_negotiated;
+  response->head.npn_negotiated_protocol =
+      response_info.npn_negotiated_protocol;
+  response->head.connection_info = response_info.connection_info;
+  response->head.was_fetched_via_proxy = request->was_fetched_via_proxy();
+  response->head.proxy_server = response_info.proxy_server;
+  response->head.socket_address = request->GetSocketAddress();
+  const content::ResourceRequestInfo* request_info =
+      content::ResourceRequestInfo::ForRequest(request);
+  if (request_info)
+    response->head.is_using_lofi = request_info->IsUsingLoFi();
+  if (ServiceWorkerRequestHandler* handler =
+          ServiceWorkerRequestHandler::GetHandler(request)) {
+    handler->GetExtraResponseInfo(&response->head);
+  }
+  AppCacheInterceptor::GetExtraResponseInfo(
+      request, &response->head.appcache_id,
+      &response->head.appcache_manifest_url);
+  if (info->is_load_timing_enabled())
+    request->GetLoadTimingInfo(&response->head.load_timing);
+
+  if (request->ssl_info().cert.get()) {
+    SSLStatus ssl_status;
+    GetSSLStatusForRequest(request->url(), request->ssl_info(),
+                           info->GetChildID(), &ssl_status);
+    response->head.security_info = SerializeSecurityInfo(ssl_status);
+  } else {
+    // We should not have any SSL state.
+    DCHECK(!request->ssl_info().cert_status);
+    DCHECK_EQ(request->ssl_info().security_bits, -1);
+    DCHECK_EQ(request->ssl_info().key_exchange_info, 0);
+    DCHECK(!request->ssl_info().connection_status);
+  }
+}
+
+DownloadInterruptReason UrlDownloader::BeginDownload(
+    DownloadManager* download_manager,
+    scoped_ptr<net::URLRequest> request,
+    const Referrer& referrer,
+    bool is_content_initiated,
+    ResourceContext* context,
+    bool prefer_cache,
+    bool do_not_prompt_for_login,
+    scoped_ptr<DownloadSaveInfo> save_info,
+    uint32 download_id,
+    const DownloadUrlParameters::OnStartedCallback& started_callback) {
+  SetReferrerForRequest(request.get(), referrer);
+
+  int extra_load_flags = net::LOAD_NORMAL;
+  if (prefer_cache) {
+    // If there is upload data attached, only retrieve from cache because there
+    // is no current mechanism to prompt the user for their consent for a
+    // re-post. For GETs, try to retrieve data from the cache and skip
+    // validating the entry if present.
+    if (request->get_upload() != NULL)
+      extra_load_flags |= net::LOAD_ONLY_FROM_CACHE;
+    else
+      extra_load_flags |= net::LOAD_PREFERRING_CACHE;
+  } else {
+    extra_load_flags |= net::LOAD_DISABLE_CACHE;
+  }
+  request->SetLoadFlags(request->load_flags() | extra_load_flags);
+
+  // We treat a download as a main frame load, and thus update the policy URL on
+  // redirects.
+  //
+  // TODO(davidben): Is this correct? If this came from a
+  // ViewHostMsg_DownloadUrl in a frame, should it have first-party URL set
+  // appropriately?
+  request->set_first_party_url_policy(
+      net::URLRequest::UPDATE_FIRST_PARTY_URL_ON_REDIRECT);
+
+  ResourceRequestInfoImpl* extra_info = new ResourceRequestInfoImpl(
+      PROCESS_TYPE_RENDERER, -1, -1,
+      -1,  // frame_tree_node_id
+      0, -1, -1,
+      false,  // is_main_frame
+      false,  // parent_is_main_frame
+      -1,     // parent_render_frame_id
+      RESOURCE_TYPE_SUB_RESOURCE, ui::PAGE_TRANSITION_LINK,
+      false,  // should_replace_current_entry
+      true,   // is_download
+      false,  // is_stream
+      true,   // allow_download
+      false,  // has_user_gesture
+      false,  // enable_load_timing
+      false,  // enable_upload_progress
+      false,  // do_not_prompt_for_login
+      blink::WebReferrerPolicyDefault, blink::WebPageVisibilityStateVisible,
+      context,
+      base::WeakPtr<ResourceMessageFilter>(),  // filter
+      false,                                   // report_raw_headers
+      true,                                    // is_async
+      false);                                  // is_using_lofi
+
+  extra_info->set_do_not_prompt_for_login(do_not_prompt_for_login);
+  extra_info->AssociateWithRequest(request.get());  // Request takes ownership.
+
+  if (request->url().SchemeIs(url::kBlobScheme))
+    return DOWNLOAD_INTERRUPT_REASON_FILE_FAILED;
+
+  extra_info->set_memory_cost(CalculateApproximateMemoryCost(request.get()));
+
+  DownloadResourceHandler* handler =
+      new DownloadResourceHandler(download_id, request.get(), started_callback,
+                                  save_info.Pass());
+  handler->set_parent_download_manager(download_manager);
+
+  // From this point forward, the |UrlDownloader| is responsible for
+  // |started_callback|.
+  linked_ptr<UrlDownloader> downloader(
+      new UrlDownloader(request.Pass(), scoped_ptr<ResourceHandler>(handler)));
+
+  downloader->Start();
+
+  g_active_downloaders.push_back(downloader);
+
+  return DOWNLOAD_INTERRUPT_REASON_NONE;
+}
+
+UrlDownloader::UrlDownloader(scoped_ptr<net::URLRequest> request,
+                             scoped_ptr<ResourceHandler> handler)
+    : request_(request.Pass()),
+      handler_(handler.Pass()),
+      weak_ptr_factory_(this) {}
+
+UrlDownloader::~UrlDownloader() {
+  handler_.reset();
+}
+
+void UrlDownloader::Start() {
+  // Give the handler a chance to delay the URLRequest from being started.
+  bool defer_start;
+  if (!handler_->OnWillStart(request_->url(), &defer_start)) {
+    request_->CancelWithError(net::ERR_ABORTED);
+    return;
+  }
+
+  DCHECK(!request_->is_pending());
+
+  if (!request_->status().is_success()) {
+    return;
+  }
+
+  request_->set_delegate(this);
+  request_->Start();
+}
+
+void UrlDownloader::OnResponseStarted(net::URLRequest* request) {
+  DVLOG(1) << "OnResponseStarted: " << request_->url().spec();
+
+  if (!request_->status().is_success()) {
+    ResponseCompleted();
+    return;
+  }
+
+  ResourceRequestInfoImpl* info =
+      ResourceRequestInfoImpl::ForRequest(request_.get());
+  scoped_refptr<ResourceResponse> response(new ResourceResponse());
+  PopulateResourceResponse(info, request_.get(), response.get());
+
+  bool defer;

[asanka, 2015/11/20 19:57:46]

Initialize. OnResponseStarted() isn't required to initialize this.

[svaldez, 2015/11/23 15:18:56]

Done.

+  if (!handler_->OnResponseStarted(response.get(), &defer)) {
+    request_->CancelWithError(net::ERR_ABORTED);
+    return;
+  }

[asanka, 2015/11/20 19:57:46]

Probably want to DCHECK(!defer) if we don't expect it to happen.

[svaldez, 2015/11/23 15:18:57]

Done.

+
+  if (request_->status().is_success())
+    StartReading(false);  // Read the first chunk.
+  else
+    ResponseCompleted();
+}
+
+void UrlDownloader::OnReadCompleted(net::URLRequest* request, int bytes_read) {
+  DVLOG(1) << "OnReadCompleted: \"" << request_->url().spec() << "\""
+           << " bytes_read = " << bytes_read;
+
+  // bytes_read == -1 always implies an error.
+  if (bytes_read == -1 || !request_->status().is_success()) {
+    ResponseCompleted();
+    return;
+  }
+
+  DCHECK(bytes_read >= 0);
+  DCHECK(request_->status().is_success());
+
+  bool defer;
+  if (!handler_->OnReadCompleted(bytes_read, &defer)) {
+    request_->CancelWithError(net::ERR_ABORTED);
+    return;
+  }
+
+  if (!request_->status().is_success())
+    return;
+
+  if (bytes_read > 0) {
+    StartReading(true);  // Read the next chunk.

[asanka, 2015/11/20 19:57:46]

Shall we call this ReadNextChunk() ?

[svaldez, 2015/11/23 15:18:56]

Using ResourceLoader naming scheme.

+  } else {
+    // URLRequest reported an EOF. Call ResponseCompleted.
+    DCHECK_EQ(0, bytes_read);
+    ResponseCompleted();
+  }
+}
+
+void UrlDownloader::StartReading(bool is_continuation) {
+  int bytes_read;
+
+  // Make sure we track the buffer in at least one place.  This ensures it gets
+  // deleted even in the case the request has already finished its job and
+  // doesn't use the buffer.
+  scoped_refptr<net::IOBuffer> buf;
+  int buf_size;
+  {

[asanka, 2015/11/20 19:57:46]

Unnecessary {

[svaldez, 2015/11/23 15:18:56]

Done.

+    if (!handler_->OnWillRead(&buf, &buf_size, -1)) {
+      request_->CancelWithError(net::ERR_ABORTED);
+      return;
+    }
+  }
+
+  DCHECK(buf.get());
+  DCHECK(buf_size > 0);
+
+  request_->Read(buf.get(), buf_size, &bytes_read);
+
+  // If IO is pending, wait for the URLRequest to call OnReadCompleted.
+  if (request_->status().is_io_pending())
+    return;
+
+  if (!is_continuation || bytes_read <= 0) {
+    OnReadCompleted(request_.get(), bytes_read);
+  } else {
+    // Else, trigger OnReadCompleted asynchronously to avoid starving the IO
+    // thread in case the URLRequest can provide data synchronously.
+    base::ThreadTaskRunnerHandle::Get()->PostTask(
+        FROM_HERE,
+        base::Bind(&UrlDownloader::OnReadCompleted,
+                   weak_ptr_factory_.GetWeakPtr(), request_.get(), bytes_read));
+  }
+}
+
+void UrlDownloader::ResponseCompleted() {
+  DVLOG(1) << "ResponseCompleted: " << request_->url().spec();
+
+  ResourceRequestInfoImpl* info =
+      ResourceRequestInfoImpl::ForRequest(request_.get());
+
+  std::string security_info;
+  const net::SSLInfo& ssl_info = request_->ssl_info();
+  if (ssl_info.cert.get() != NULL) {
+    SSLStatus ssl_status;
+    GetSSLStatusForRequest(request_->url(), ssl_info, info->GetChildID(),
+                           &ssl_status);
+
+    security_info = SerializeSecurityInfo(ssl_status);
+  }
+
+  bool defer;
+  { handler_->OnResponseCompleted(request_->status(), security_info, &defer); }

[asanka, 2015/11/20 19:57:46]

Unnecessary {

[svaldez, 2015/11/23 15:18:56]

Done.

+}
+
+}  // namespace content