Initial implementation for CertNetFetcher.

net/cert_net/cert_net_fetcher_impl.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/cert_net/cert_net_fetcher_impl.h"
+
+#include "base/callback_helpers.h"
+#include "base/logging.h"
+#include "base/numerics/safe_math.h"
+#include "base/stl_util.h"
+#include "base/timer/timer.h"
+#include "net/base/load_flags.h"
+#include "net/url_request/redirect_info.h"
+#include "net/url_request/url_request_context.h"
+
+// TODO(eroman): Add support for POST parameters.
+// TODO(eroman): Add controls for bypassing the cache.
+// TODO(eroman): Add a maximum number of in-flight jobs/requests.
+// TODO(eroman): Add NetLog integration.
+
+namespace net {
+
+namespace {
+
+// The size of the buffer used for reading the response body of the URLRequest.
+const int kReadBufferSizeInBytes = 4096;
+
+// The maximum size in bytes for the response body when fetching a CRL.
+const int kMaxResponseSizeInBytesForCrl = 5 * 1024 * 1024;
+
+// The maximum size in bytes for the response body when fetching an AIA URL
+// (caIssuers/OCSP).
+const int kMaxResponseSizeInBytesForAia = 64 * 1024;
+
+// The default timeout in seconds for fetch requests.
+const int kTimeoutSeconds = 15;
+
+// Policy for which URLs are allowed to be fetched. This is called both for the
+// initial URL and for each redirect. Returns OK on success or a net error
+// code on failure.
+Error CanFetchUrl(const GURL& url) {
+  if (!url.SchemeIs("http"))
+    return ERR_DISALLOWED_URL_SCHEME;
+  return OK;
+}
+
+base::TimeDelta GetTimeout(int timeout_milliseconds) {
+  if (timeout_milliseconds == CertNetFetcher::DEFAULT)
+    return base::TimeDelta::FromSeconds(kTimeoutSeconds);
+  return base::TimeDelta::FromMilliseconds(timeout_milliseconds);
+}
+
+size_t GetMaxResponseBytes(int max_response_bytes,
+                           size_t default_max_response_bytes) {
+  if (max_response_bytes == CertNetFetcher::DEFAULT)
+    return default_max_response_bytes;
+
+  // Ensure that the specified limit is not negative, and cannot result in an
+  // overflow while reading.
+  base::CheckedNumeric<size_t> check(max_response_bytes);
+  check += kReadBufferSizeInBytes;
+  DCHECK(check.IsValid());
+
+  return max_response_bytes;
+}
+
+enum HttpMethod {
+  HTTP_METHOD_GET,
+  HTTP_METHOD_POST,
+};
+
+}  // namespace
+
+// CertNetFetcherImpl::RequestImpl tracks an outstanding call to Fetch().
+class CertNetFetcherImpl::RequestImpl : public CertNetFetcher::Request {
+ public:
+  explicit RequestImpl(const FetchCallback& callback)
+      : callback_(callback), job_(nullptr) {
+    DCHECK(!callback.is_null());
+  }
+
+  // Deletion cancels the outstanding request.
+  ~RequestImpl() override;
+
+  void OnAttachedToJob(Job* job) {
+    DCHECK(!job_);
+    job_ = job;
+  }
+
+  void OnJobCancelled(Job* job) {
+    DCHECK_EQ(job_, job);
+    job_ = nullptr;
+    callback_.Reset();
+  }
+
+  void OnJobCompleted(Job* job,
+                      Error error,
+                      const std::vector<uint8_t>& response_body) {
+    DCHECK_EQ(job_, job);
+    job_ = nullptr;
+    base::ResetAndReturn(&callback_).Run(error, response_body);
+  }
+
+ private:
+  // The callback to invoke when the request has completed.
+  FetchCallback callback_;
+
+  // A non-owned pointer to the job that is executing the request.
+  Job* job_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(RequestImpl);
+};
+
+struct CertNetFetcherImpl::RequestParams {
+  RequestParams();
+
+  bool operator<(const RequestParams& other) const;
+
+  GURL url;
+  HttpMethod http_method;
+  size_t max_response_bytes;
+
+  // If set to a value <= 0 then means "no timeout".
+  base::TimeDelta timeout;
+
+  // IMPORTANT: When adding fields to this structure, update operator<().
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(RequestParams);
+};
+
+CertNetFetcherImpl::RequestParams::RequestParams()
+    : http_method(HTTP_METHOD_GET), max_response_bytes(0) {
+}
+
+bool CertNetFetcherImpl::RequestParams::operator<(
+    const RequestParams& other) const {
+  if (url != other.url)
+    return url < other.url;
+  if (http_method != other.http_method)
+    return http_method < other.http_method;
+  if (max_response_bytes != other.max_response_bytes)
+    return max_response_bytes < other.max_response_bytes;
+  return timeout < other.timeout;
+}
+
+// CertNetFetcherImpl::Job tracks an outstanding URLRequest as well as all of
+// the pending requests for it.
+class CertNetFetcherImpl::Job : public URLRequest::Delegate {
+ public:
+  Job(scoped_ptr<RequestParams> request_params, CertNetFetcherImpl* parent);
+  ~Job() override;
+
+  // Cancels the job and all requests attached to it. No callbacks will be
+  // invoked following cancellation.
+  void Cancel();
+
+  const RequestParams& request_params() const { return *request_params_; }
+
+  // Attaches a request to the job. When the job completes it will notify the
+  // request of completion through OnJobCompleted. Note that the Job does NOT
+  // own the request.
+  void AttachRequest(RequestImpl* request);
+
+  // Removes |request| from the job.
+  void DetachRequest(RequestImpl* request);
+
+  // Creates and starts a URLRequest for the job. After the request has
+  // completed, OnJobCompleted() will be invoked and all the registered requests
+  // notified of completion.
+  void StartURLRequest(URLRequestContext* context);
+
+ private:
+  // The pointers in RequestList are not owned by the Job.
+  using RequestList = std::vector<RequestImpl*>;
+
+  // Implementation of URLRequest::Delegate
+  void OnReceivedRedirect(URLRequest* request,
+                          const RedirectInfo& redirect_info,
+                          bool* defer_redirect) override;
+  void OnResponseStarted(URLRequest* request) override;
+  void OnReadCompleted(URLRequest* request, int bytes_read) override;
+
+  // Clears the URLRequest and timer. Helper for doing work common to
+  // cancellation and job completion.
+  void Stop();
+
+  // Reads as much data as available from |request|.
+  void ReadBody(URLRequest* request);
+
+  // Helper to copy the partial bytes read from the read IOBuffer to an
+  // aggregated buffer.
+  bool ConsumeBytesRead(URLRequest* request, int num_bytes);
+
+  // Called once the job has exceeded its deadline.
+  void OnTimeout();
+
+  // Called when the URLRequest has completed (either success or failure).
+  void OnUrlRequestCompleted(URLRequest* request);
+
+  // Called when the Job has completed. The job may finish in response to a
+  // timeout, an invalid URL, or the URLRequest completing. By the time this
+  // method is called, the response variables have been assigned
+  // (result_net_error_ and response_body_).
+  void OnJobCompleted();
+
+  // The requests attached to this job.
+  RequestList requests_;
+
+  // The input parameters for starting a URLRequest.
+  scoped_ptr<RequestParams> request_params_;
+
+  // The URLRequest response information.
+  std::vector<uint8_t> response_body_;
+  Error result_net_error_;
+
+  scoped_ptr<URLRequest> url_request_;
+  scoped_refptr<IOBuffer> read_buffer_;
+
+  // Used to timeout the job when the URLRequest takes too long. This timer is
+  // also used for notifying a failure to start the URLRequest.
+  base::OneShotTimer<Job> timer_;
+
+  // Non-owned pointer to the CertNetFetcherImpl that created this job.
+  CertNetFetcherImpl* parent_;
+
+  DISALLOW_COPY_AND_ASSIGN(Job);
+};
+
+CertNetFetcherImpl::RequestImpl::~RequestImpl() {
+  if (job_)
+    job_->DetachRequest(this);
+}
+
+CertNetFetcherImpl::Job::Job(scoped_ptr<RequestParams> request_params,
+                             CertNetFetcherImpl* parent)
+    : request_params_(request_params.Pass()),
+      result_net_error_(ERR_IO_PENDING),
+      parent_(parent) {
+}
+
+CertNetFetcherImpl::Job::~Job() {
+  Cancel();
+}
+
+void CertNetFetcherImpl::Job::Cancel() {
+  parent_ = nullptr;
+
+  for (RequestImpl* request : requests_)
+    request->OnJobCancelled(this);
+  requests_.clear();
+
+  Stop();
+}
+
+void CertNetFetcherImpl::Job::AttachRequest(RequestImpl* request) {
+  // Shouldn't already be in the job.
+  DCHECK(std::find(requests_.begin(), requests_.end(), request) ==
+         requests_.end());
+
+  requests_.push_back(request);
+  request->OnAttachedToJob(this);
+}
+
+void CertNetFetcherImpl::Job::DetachRequest(RequestImpl* request) {
+  scoped_ptr<Job> delete_this;
+
+  RequestList::iterator it =
+      std::find(requests_.begin(), requests_.end(), request);
+  CHECK(it != requests_.end());
+  requests_.erase(it);
+
+  // If there are no longer any requests attached to the job then
+  // cancel and delete it.
+  if (requests_.empty() && parent_)

[davidben, 2015/04/07 19:22:25]

Is it possible for parent_ to be null here? Right after we null parent_, we
clear through all the requests, which means DetachRequest can't have been
called.

[eroman, 2015/04/07 20:22:40]

Done.

+    delete_this = parent_->RemoveJob(this);
+}
+
+void CertNetFetcherImpl::Job::StartURLRequest(URLRequestContext* context) {
+  Error error = CanFetchUrl(request_params_->url);
+  if (error != OK) {
+    result_net_error_ = error;
+    // The CertNetFetcher's API contract is that requests always complete
+    // asynchronously. Use the timer class so the task is easily cancelled.
+    timer_.Start(FROM_HERE, base::TimeDelta(), this, &Job::OnJobCompleted);
+    return;
+  }
+
+  // Start the URLRequest.
+  read_buffer_ = new IOBuffer(kReadBufferSizeInBytes);
+  url_request_ =
+      context->CreateRequest(request_params_->url, DEFAULT_PRIORITY, this);
+  if (request_params_->http_method == HTTP_METHOD_POST)
+    url_request_->set_method("POST");
+  url_request_->SetLoadFlags(LOAD_DO_NOT_SAVE_COOKIES |
+                             LOAD_DO_NOT_SEND_COOKIES);
+  url_request_->Start();
+
+  // Start a timer to limit how long the job runs for.
+  if (request_params_->timeout > base::TimeDelta())
+    timer_.Start(FROM_HERE, request_params_->timeout, this, &Job::OnTimeout);
+}
+
+void CertNetFetcherImpl::Job::OnReceivedRedirect(
+    URLRequest* request,
+    const RedirectInfo& redirect_info,
+    bool* defer_redirect) {
+  DCHECK_EQ(url_request_.get(), request);
+
+  // Ensure that the new URL matches the policy.
+  Error error = CanFetchUrl(redirect_info.new_url);
+  if (error != OK) {
+    request->CancelWithError(error);
+    OnUrlRequestCompleted(request);
+    return;
+  }
+}
+
+void CertNetFetcherImpl::Job::OnResponseStarted(URLRequest* request) {
+  DCHECK_EQ(url_request_.get(), request);
+
+  if (!request->status().is_success()) {
+    OnUrlRequestCompleted(request);
+    return;
+  }
+
+  if (request->GetResponseCode() != 200) {
+    // TODO(eroman): Use a more specific error code.
+    request->CancelWithError(ERR_FAILED);
+    OnUrlRequestCompleted(request);
+    return;
+  }
+
+  ReadBody(request);
+}
+
+void CertNetFetcherImpl::Job::OnReadCompleted(URLRequest* request,
+                                              int bytes_read) {
+  DCHECK_EQ(url_request_.get(), request);
+
+  // Keep reading the response body.
+  if (ConsumeBytesRead(request, bytes_read))
+    ReadBody(request);
+}
+
+void CertNetFetcherImpl::Job::Stop() {
+  timer_.Stop();
+  url_request_.reset();
+}
+
+void CertNetFetcherImpl::Job::ReadBody(URLRequest* request) {
+  // Read as many bytes as are available synchronously.
+  int num_bytes;
+  while (
+      request->Read(read_buffer_.get(), kReadBufferSizeInBytes, &num_bytes)) {
+    if (!ConsumeBytesRead(request, num_bytes))
+      return;
+  }
+
+  // Check whether the read failed synchronously.
+  if (!request->status().is_io_pending())
+    OnUrlRequestCompleted(request);
+  return;
+}
+
+bool CertNetFetcherImpl::Job::ConsumeBytesRead(URLRequest* request,
+                                               int num_bytes) {
+  if (num_bytes <= 0) {
+    // Error while reading, or EOF.
+    OnUrlRequestCompleted(request);
+    return false;
+  }
+
+  // Enforce maximum size bound.
+  if (num_bytes + response_body_.size() > request_params_->max_response_bytes) {
+    request->CancelWithError(ERR_FILE_TOO_BIG);
+    OnUrlRequestCompleted(request);
+    return false;
+  }
+
+  // Append the data to |response_body_|.
+  response_body_.insert(response_body_.end(), read_buffer_->data(),
+                        read_buffer_->data() + num_bytes);
+  return true;
+}
+
+void CertNetFetcherImpl::Job::OnTimeout() {
+  result_net_error_ = ERR_TIMED_OUT;
+  url_request_->CancelWithError(result_net_error_);
+  OnJobCompleted();
+}
+
+void CertNetFetcherImpl::Job::OnUrlRequestCompleted(URLRequest* request) {
+  DCHECK_EQ(request, url_request_.get());
+
+  if (request->status().is_success())
+    result_net_error_ = OK;
+  else
+    result_net_error_ = static_cast<Error>(request->status().error());
+
+  OnJobCompleted();
+}
+
+void CertNetFetcherImpl::Job::OnJobCompleted() {
+  // Stop the timer and clear the URLRequest.
+  Stop();
+
+  // Invoking the callbacks is subtle as state may be mutated while iterating
+  // through the callbacks:
+  //
+  //   * The parent CertNetFetcherImpl may be deleted
+  //   * Requests in this job may be cancelled
+
+  scoped_ptr<Job> delete_this = parent_->RemoveJob(this);
+  parent_->SetCurrentlyCompletingJob(this);
+
+  while (!requests_.empty()) {
+    RequestImpl* request = requests_.front();
+    requests_.erase(requests_.begin());

[davidben, 2015/04/07 19:22:25]

Probably doesn't matter, but note this loop is O(N^2). (If we want it linear,
this could be a std::list and each Request could hold onto an iterator into
requests_ so that removal of a single request is also O(1). Another option is a
std::set, though then you don't get FIFO ordering.)

[eroman, 2015/04/07 19:54:54]

The simplest fix would be to use a dequeue instead of a vector.

Alternately I could use an std::vector and pop elements off the back (perhaps
having reversed it first).

I am not sure how important FIFO completion is. Likely doesn't matter but kind
of makes more sense for things to complete in order.

[eroman, 2015/04/07 20:22:40]

Done (switched to deque).

[davidben, 2015/04/07 21:22:39]

The reason I suggested a list was that a deque still makes canceling N requests
attached to a job take O(N^2). I doubt this really matters though, so *shrug*

[eroman, 2015/04/08 16:23:39]

Will leave it as is for now along with a comment.
I don't think this will matter in practice, but can do a follow-up after some
testing.

+    request->OnJobCompleted(this, result_net_error_, response_body_);
+  }
+
+  if (parent_)
+    parent_->ClearCurrentlyCompletingJob(this);
+}
+
+CertNetFetcherImpl::CertNetFetcherImpl(URLRequestContext* context)
+    : currently_completing_job_(nullptr), context_(context) {
+}
+
+CertNetFetcherImpl::~CertNetFetcherImpl() {
+  STLDeleteElements(&jobs_);

[davidben, 2015/04/07 19:22:25]

Maybe add a comment here like:

  // The CertNetFetcher was destroyed in a FetchCallback. Detach all remaining
requests from the job so no further callbacks are called.

[eroman, 2015/04/07 20:22:40]

Done.

+  if (currently_completing_job_)
+    currently_completing_job_->Cancel();
+}
+
+scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::FetchCaIssuers(
+    const GURL& url,
+    int timeout_milliseconds,
+    int max_response_bytes,
+    const FetchCallback& callback) {
+  scoped_ptr<RequestParams> request_params(new RequestParams);
+
+  request_params->url = url;
+  request_params->http_method = HTTP_METHOD_GET;
+  request_params->timeout = GetTimeout(timeout_milliseconds);
+  request_params->max_response_bytes =
+      GetMaxResponseBytes(max_response_bytes, kMaxResponseSizeInBytesForAia);
+
+  return Fetch(request_params.Pass(), callback);
+}
+
+scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::FetchCrl(
+    const GURL& url,
+    int timeout_milliseconds,
+    int max_response_bytes,
+    const FetchCallback& callback) {
+  scoped_ptr<RequestParams> request_params(new RequestParams);
+
+  request_params->url = url;
+  request_params->http_method = HTTP_METHOD_GET;
+  request_params->timeout = GetTimeout(timeout_milliseconds);
+  request_params->max_response_bytes =
+      GetMaxResponseBytes(max_response_bytes, kMaxResponseSizeInBytesForCrl);
+
+  return Fetch(request_params.Pass(), callback);
+}
+
+scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::FetchOcsp(
+    const GURL& url,
+    int timeout_milliseconds,
+    int max_response_bytes,
+    const FetchCallback& callback) {
+  scoped_ptr<RequestParams> request_params(new RequestParams);
+
+  request_params->url = url;
+  request_params->http_method = HTTP_METHOD_GET;
+  request_params->timeout = GetTimeout(timeout_milliseconds);
+  request_params->max_response_bytes =
+      GetMaxResponseBytes(max_response_bytes, kMaxResponseSizeInBytesForAia);
+
+  return Fetch(request_params.Pass(), callback);
+}
+
+bool CertNetFetcherImpl::JobComparator::operator()(const Job* job1,
+                                                   const Job* job2) const {
+  return job1->request_params() < job2->request_params();
+}
+
+scoped_ptr<CertNetFetcher::Request> CertNetFetcherImpl::Fetch(
+    scoped_ptr<RequestParams> request_params,
+    const FetchCallback& callback) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // If there is an in-progress job that matches the request parameters use it.
+  // Otherwise start a new job.
+  Job* job = FindJob(*request_params);
+
+  if (!job) {
+    job = new Job(request_params.Pass(), this);
+    jobs_.insert(job);
+    job->StartURLRequest(context_);
+  }
+
+  scoped_ptr<RequestImpl> request(new RequestImpl(callback));
+  job->AttachRequest(request.get());
+  return request.Pass();

[davidben, 2015/04/07 19:22:25]

Perhaps this trio could be something like

  return job->CreateRequest(callback);

Since a RequestImpl always gets OnAttachedToJob called immediately after
creation anyway. It may as well be a parameter in the constructor.

[eroman, 2015/04/07 20:22:40]

Done.

+}
+
+struct CertNetFetcherImpl::JobToRequestParamsComparator {
+  bool operator()(const Job* job,
+                  const CertNetFetcherImpl::RequestParams& value) const {
+    return job->request_params() < value;
+  }
+};
+
+CertNetFetcherImpl::Job* CertNetFetcherImpl::FindJob(
+    const RequestParams& params) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // The JobSet is kept in sorted order so items can be found using binary
+  // search.
+  JobSet::iterator it = std::lower_bound(jobs_.begin(), jobs_.end(), params,
+                                         JobToRequestParamsComparator());
+  if (it != jobs_.end() && !(params < (*it)->request_params()))
+    return *it;
+  return nullptr;
+}
+
+scoped_ptr<CertNetFetcherImpl::Job> CertNetFetcherImpl::RemoveJob(Job* job) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  bool erased_job = jobs_.erase(job) == 1;
+  DCHECK(erased_job);
+  return make_scoped_ptr(job);
+}
+
+void CertNetFetcherImpl::SetCurrentlyCompletingJob(Job* job) {
+  DCHECK(!currently_completing_job_);
+  DCHECK(job);
+  currently_completing_job_ = job;
+}
+
+void CertNetFetcherImpl::ClearCurrentlyCompletingJob(Job* job) {
+  DCHECK_EQ(currently_completing_job_, job);
+  currently_completing_job_ = nullptr;
+}
+
+}  // namespace net