DevTools protocol interception, blocking & modification of requests

content/browser/devtools/devtools_url_interceptor_request_job.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/devtools/devtools_url_interceptor_request_job.h"
+
+#include "base/memory/ptr_util.h"
+#include "base/strings/stringprintf.h"
+#include "content/browser/devtools/protocol/network_handler.h"
+#include "net/base/elements_upload_data_stream.h"
+#include "net/base/io_buffer.h"
+#include "net/base/upload_bytes_element_reader.h"
+#include "net/base/upload_element_reader.h"
+#include "net/cert/cert_status_flags.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_util.h"
+#include "net/url_request/url_request_context.h"
+
+namespace content {
+
+using CommandStatus = DevToolsURLRequestInterceptor::CommandStatus;
+
+namespace {
+class ProxyUploadBytesElementReader : public net::UploadElementReader {

[Sami, 2017/05/19 17:15:43]

bikeshed: ProxyUploadElementReader? (not sure what "bytes" means here)

[alex clarke (OOO till 29th), 2017/05/23 15:46:55]

Done.

+ public:
+  explicit ProxyUploadBytesElementReader(net::UploadElementReader* reader)
+      : reader_(reader) {}
+
+  ~ProxyUploadBytesElementReader() override {}
+
+  // net::UploadElementReader overrides:
+  int Init(const net::CompletionCallback& callback) override {
+    return reader_->Init(callback);
+  }
+
+  uint64_t GetContentLength() const override {
+    return reader_->GetContentLength();
+  }
+
+  uint64_t BytesRemaining() const override { return reader_->BytesRemaining(); }
+
+  bool IsInMemory() const override { return reader_->IsInMemory(); }
+
+  int Read(net::IOBuffer* buf,
+           int buf_length,
+           const net::CompletionCallback& callback) override {
+    return reader_->Read(buf, buf_length, callback);
+  }
+
+ private:
+  net::UploadElementReader* reader_;  // NOT OWNED
+
+  DISALLOW_COPY_AND_ASSIGN(ProxyUploadBytesElementReader);
+};
+
+std::unique_ptr<net::UploadElementReader> GetUploadData(
+    net::URLRequest* request) {
+  if (!request->has_upload())
+    return nullptr;
+
+  const net::UploadDataStream* stream = request->get_upload();
+  if (!stream->GetElementReaders())
+    return nullptr;
+
+  DCHECK_EQ(1u, stream->GetElementReaders()->size());
+  return base::MakeUnique<ProxyUploadBytesElementReader>(
+      (*stream->GetElementReaders())[0].get());
+}
+}  // namespace
+
+DevToolsURLInterceptorRequestJob::DevToolsURLInterceptorRequestJob(
+    base::WeakPtr<DevToolsURLRequestInterceptor::State>
+        devtools_url_request_interceptor_state,
+    const std::string& intercept_id,
+    net::URLRequest* original_request,
+    net::NetworkDelegate* original_network_delegate,
+    base::WeakPtr<protocol::NetworkHandler> network_handler,
+    bool is_redirect)
+    : net::URLRequestJob(original_request, original_network_delegate),
+      devtools_url_request_interceptor_state_(
+          devtools_url_request_interceptor_state),
+      request_details_(original_request->url(),
+                       original_request->method(),
+                       GetUploadData(original_request),
+                       original_request->extra_request_headers(),
+                       original_request->priority(),
+                       original_request->context()),
+      intercept_id_(intercept_id),
+      network_handler_(network_handler),
+      io_thread_task_runner_(
+          BrowserThread::GetTaskRunnerForThread(BrowserThread::IO)),
+      is_redirect_(is_redirect),
+      weak_ptr_factory_(this) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  devtools_url_request_interceptor_state_->RegisterJob(this, intercept_id_);
+}
+
+DevToolsURLInterceptorRequestJob::~DevToolsURLInterceptorRequestJob() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (devtools_url_request_interceptor_state_) {
+    devtools_url_request_interceptor_state_->UnregisterJob(intercept_id_);
+  }
+}
+
+// net::URLRequestJob implementation:
+void DevToolsURLInterceptorRequestJob::SetExtraRequestHeaders(
+    const net::HttpRequestHeaders& headers) {
+  request_details_.extra_request_headers = headers;
+}
+
+void DevToolsURLInterceptorRequestJob::Start() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (is_redirect_) {
+    // If this is a fetch in response to a redirect, we have already sent the
+    // Network.InterceptedRedirectEvent and the user opted to allow it so
+    // there's no need to send Network.InterceptedRequestEvent. We can just
+    // start the SubRequest.
+    sub_request_.reset(new SubRequest(request_details_, this));
+  } else {
+    waiting_for_user_response_ = true;
+    BrowserThread::PostTask(
+        BrowserThread::UI, FROM_HERE,
+        base::Bind(&DevToolsURLInterceptorRequestJob::
+                       SendInterceptedRequestEventOnUiThread,
+                   weak_ptr_factory_.GetWeakPtr()));
+  }
+}
+
+void DevToolsURLInterceptorRequestJob::SendInterceptedRequestEventOnUiThread() {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  if (network_handler_) {
+    network_handler_->frontend()->InterceptedRequest(
+        intercept_id_,
+        protocol::NetworkHandler::CreateRequestFromURLRequest(request()));
+  }
+}
+
+void DevToolsURLInterceptorRequestJob::Kill() {
+  if (sub_request_)
+    sub_request_->Cancel();
+
+  URLRequestJob::Kill();
+}
+
+int DevToolsURLInterceptorRequestJob::ReadRawData(net::IOBuffer* buf,
+                                                  int buf_size) {
+  if (sub_request_) {
+    int size = sub_request_->request()->Read(buf, buf_size);
+    return size;
+  } else {
+    CHECK(mock_response_details_);
+    return mock_response_details_->ReadRawData(buf, buf_size);
+  }
+}
+
+int DevToolsURLInterceptorRequestJob::GetResponseCode() const {
+  if (sub_request_) {
+    return sub_request_->request()->GetResponseCode();
+  } else {
+    CHECK(mock_response_details_);
+    return mock_response_details_->response_headers()->response_code();
+  }
+}
+
+void DevToolsURLInterceptorRequestJob::GetResponseInfo(
+    net::HttpResponseInfo* info) {
+  // NOTE this can get called during URLRequestJob::NotifyStartError in which
+  // case we might not have either a sub request or a mock response.
+  if (sub_request_) {
+    *info = sub_request_->request()->response_info();
+  } else if (mock_response_details_) {
+    info->headers = mock_response_details_->response_headers().get();
+  }
+}
+
+const net::HttpResponseHeaders*
+DevToolsURLInterceptorRequestJob::GetHttpResponseHeaders() const {
+  if (sub_request_) {
+    net::URLRequest* request = sub_request_->request();
+    return request->response_info().headers.get();
+  }
+  CHECK(mock_response_details_);
+  return mock_response_details_->response_headers().get();
+}
+
+bool DevToolsURLInterceptorRequestJob::GetMimeType(
+    std::string* mime_type) const {
+  const net::HttpResponseHeaders* response_headers = GetHttpResponseHeaders();
+  if (!response_headers)
+    return false;
+  return response_headers->GetMimeType(mime_type);
+}
+
+bool DevToolsURLInterceptorRequestJob::GetCharset(std::string* charset) {
+  const net::HttpResponseHeaders* response_headers = GetHttpResponseHeaders();
+  if (!response_headers)
+    return false;
+  return response_headers->GetCharset(charset);
+}
+
+void DevToolsURLInterceptorRequestJob::GetLoadTimingInfo(
+    net::LoadTimingInfo* load_timing_info) const {
+  if (sub_request_) {
+    sub_request_->request()->GetLoadTimingInfo(load_timing_info);
+  } else {
+    CHECK(mock_response_details_);
+    // TODO(alexclarke): Investigate setting the other members too where
+    // possible.
+    load_timing_info->receive_headers_end =
+        mock_response_details_->response_time();
+  }
+}
+
+void DevToolsURLInterceptorRequestJob::OnAuthRequired(
+    net::URLRequest* request,
+    net::AuthChallengeInfo* auth_info) {
+  DCHECK(sub_request_);
+  DCHECK_EQ(request, sub_request_->request());
+  // TODO(alexclarke): Implement this.
+  LOG(WARNING) << "Auth required to fetch URL, aborting.";
+  request->CancelAuth();
+  DispatchError(net::ERR_NOT_IMPLEMENTED);
+}
+
+void DevToolsURLInterceptorRequestJob::OnResponseStarted(
+    net::URLRequest* request,
+    int net_error) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(sub_request_);
+  DCHECK_EQ(request, sub_request_->request());
+  DCHECK_NE(net::ERR_IO_PENDING, net_error);
+
+  if (net_error != net::OK) {
+    sub_request_->Cancel();
+    DispatchError(static_cast<net::Error>(net_error));
+    return;
+  }
+
+  NotifyHeadersComplete();
+}
+
+void DevToolsURLInterceptorRequestJob::OnReadCompleted(net::URLRequest* request,
+                                                       int num_bytes) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK_EQ(request, sub_request_->request());
+
+  ReadRawDataComplete(num_bytes);
+}
+
+void DevToolsURLInterceptorRequestJob::OnReceivedRedirect(
+    net::URLRequest* request,
+    const net::RedirectInfo& redirectinfo,
+    bool* defer_redirect) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  *defer_redirect = true;
+
+  size_t iter = 0;
+  std::string header_name;
+  std::string header_value;
+  std::unique_ptr<protocol::DictionaryValue> headers_dict(
+      protocol::DictionaryValue::create());
+  while (request->response_headers()->EnumerateHeaderLines(&iter, &header_name,
+                                                           &header_value)) {
+    headers_dict->setString(header_name, header_value);
+  }
+
+  redirect_.reset(new net::RedirectInfo(redirectinfo));
+  sub_request_->Cancel();
+  sub_request_.reset();
+
+  waiting_for_user_response_ = true;
+  BrowserThread::PostTask(
+      BrowserThread::UI, FROM_HERE,
+      base::Bind(&DevToolsURLInterceptorRequestJob::
+                     SendInterceptedRedirectEventOnUiThread,
+                 weak_ptr_factory_.GetWeakPtr(),
+                 base::Owned(headers_dict.release()), redirectinfo.status_code,
+                 redirectinfo.new_url.spec()));
+}
+
+void DevToolsURLInterceptorRequestJob::SendInterceptedRedirectEventOnUiThread(
+    protocol::DictionaryValue* headers_dict,
+    int http_status_code,
+    std::string redirect_url) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  return network_handler_->frontend()->InterceptedRedirect(
+      intercept_id_, protocol::Object::fromValue(headers_dict, nullptr),
+      http_status_code, redirect_url);
+}
+
+void DevToolsURLInterceptorRequestJob::OnSSLCertificateError(
+    net::URLRequest* request,
+    const net::SSLInfo& ssl_info,
+    bool fatal) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(sub_request_);
+  DCHECK_EQ(request, sub_request_->request());
+
+  // Revocation check failures are not fatal.
+  if (net::IsCertStatusMinorError(ssl_info.cert_status)) {
+    request->ContinueDespiteLastError();
+    return;
+  }
+  LOG(WARNING) << "SSL certificate error, aborting.";
+
+  // Certificate errors are in same space as net errors.
+  sub_request_->Cancel();
+  DispatchError(static_cast<net::Error>(
+      net::MapCertStatusToNetError(ssl_info.cert_status)));
+}
+
+CommandStatus DevToolsURLInterceptorRequestJob::AllowRequest() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (!waiting_for_user_response_)
+    return CommandStatus::CommandAlreadyProcessed;
+  waiting_for_user_response_ = false;
+
+  if (redirect_) {
+    // NOTE we don't append the text form of the status code because
+    // net::HttpResponseHeaders doesn't need that.
+    std::string raw_headers =
+        base::StringPrintf("HTTP/1.1 %d", redirect_->status_code);
+    raw_headers.append(1, '\0');
+    raw_headers.append("Location: ");
+    raw_headers.append(redirect_->new_url.spec());
+    raw_headers.append(2, '\0');
+    mock_response_details_.reset(new MockResponseDetails(
+        make_scoped_refptr(new net::HttpResponseHeaders(raw_headers)), "", 0,
+        base::TimeTicks::Now()));
+    redirect_.reset();
+
+    DCHECK(devtools_url_request_interceptor_state_);
+    devtools_url_request_interceptor_state_->ExpectRequestAfterRedirect(
+        request(), intercept_id_);
+    NotifyHeadersComplete();
+  } else {
+    // The reason we start a sub request is because we are in full control of it
+    // and can choose to ignore it if, for example, the fetch encounters a
+    // redirect that the user chooses to replace with a mock response.
+    sub_request_.reset(new SubRequest(request_details_, this));
+  }
+  return CommandStatus::OK;
+}
+
+CommandStatus DevToolsURLInterceptorRequestJob::BlockRequest(
+    net::Error error_reason) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (!waiting_for_user_response_)
+    return CommandStatus::CommandAlreadyProcessed;
+  waiting_for_user_response_ = false;
+
+  DispatchError(error_reason);
+  return CommandStatus::OK;
+}
+
+CommandStatus DevToolsURLInterceptorRequestJob::ModifyRequest(
+    std::unique_ptr<DevToolsURLRequestInterceptor::Modifications>
+        modifications) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (!waiting_for_user_response_)
+    return CommandStatus::CommandAlreadyProcessed;
+  waiting_for_user_response_ = false;
+
+  // Note this redirect is not visible to the caller by design. If they want a
+  // visible redirect they can mock a response with a 302.
+  if (modifications->url.isJust())
+    request_details_.url = GURL(modifications->url.fromJust());
+
+  if (modifications->method.isJust())
+    request_details_.method = modifications->method.fromJust();
+
+  if (modifications->post_data.isJust()) {
+    const std::string& post_data = modifications->post_data.fromJust();
+    std::vector<char> data(post_data.begin(), post_data.end());
+    request_details_.post_data.reset(
+        new net::UploadOwnedBytesElementReader(&data));
+  }
+
+  if (modifications->headers.isJust()) {
+    request_details_.extra_request_headers.Clear();
+    std::unique_ptr<protocol::DictionaryValue> headers =
+        modifications->headers.fromJust()->toValue();
+    for (size_t i = 0; i < headers->size(); i++) {
+      std::string value;
+      if (headers->at(i).second->asString(&value)) {
+        request_details_.extra_request_headers.SetHeader(headers->at(i).first,
+                                                         value);
+      }
+    }
+  }
+
+  sub_request_.reset(new SubRequest(request_details_, this));
+  return CommandStatus::OK;
+}
+
+CommandStatus DevToolsURLInterceptorRequestJob::MockResponse(
+    std::string raw_response) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (!waiting_for_user_response_)
+    return CommandStatus::CommandAlreadyProcessed;
+  waiting_for_user_response_ = false;
+
+  mock_response_details_.reset(
+      new MockResponseDetails(std::move(raw_response), base::TimeTicks::Now()));
+
+  std::string value;
+  if (mock_response_details_->response_headers()->IsRedirect(&value)) {
+    DCHECK(devtools_url_request_interceptor_state_);
+    devtools_url_request_interceptor_state_->ExpectRequestAfterRedirect(
+        request(), intercept_id_);
+  }
+  NotifyHeadersComplete();
+  return CommandStatus::OK;
+}
+
+void DevToolsURLInterceptorRequestJob::DispatchError(net::Error reason) {
+  DCHECK_NE(reason, net::OK);
+  io_thread_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&DevToolsURLInterceptorRequestJob::NotifyStartError,
+                 weak_ptr_factory_.GetWeakPtr(),
+                 net::URLRequestStatus(net::URLRequestStatus::FAILED, reason)));
+}
+
+DevToolsURLInterceptorRequestJob::RequestDetails::RequestDetails(
+    const GURL& url,
+    const std::string& method,
+    std::unique_ptr<net::UploadElementReader> post_data,
+    const net::HttpRequestHeaders& extra_request_headers,
+    const net::RequestPriority& priority,
+    const net::URLRequestContext* url_request_context)
+    : url(url),
+      method(method),
+      post_data(std::move(post_data)),
+      extra_request_headers(extra_request_headers),
+      priority(priority),
+      url_request_context(url_request_context) {}
+
+DevToolsURLInterceptorRequestJob::RequestDetails::~RequestDetails() {}
+
+DevToolsURLInterceptorRequestJob::SubRequest::SubRequest(
+    DevToolsURLInterceptorRequestJob::RequestDetails& request_details,
+    DevToolsURLInterceptorRequestJob* devtools_interceptor_request_job)
+    : devtools_interceptor_request_job_(devtools_interceptor_request_job),
+      devtools_url_request_interceptor_state_(
+          devtools_interceptor_request_job_
+              ->devtools_url_request_interceptor_state_),
+      fetch_in_progress_(true) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  request_ = request_details.url_request_context->CreateRequest(
+      request_details.url, request_details.priority,
+      devtools_interceptor_request_job_),
+  request_->set_method(request_details.method);
+  request_->SetExtraRequestHeaders(request_details.extra_request_headers);
+
+  if (request_details.post_data) {
+    request_->set_upload(net::ElementsUploadDataStream::CreateWithReader(
+        std::move(request_details.post_data), 0));
+  }
+
+  if (!devtools_url_request_interceptor_state_)
+    return;
+  devtools_url_request_interceptor_state_->RegisterSubRequest(request_.get());
+  request_->Start();
+}
+
+DevToolsURLInterceptorRequestJob::SubRequest::~SubRequest() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  if (!devtools_url_request_interceptor_state_)
+    return;
+  devtools_url_request_interceptor_state_->UnregisterSubRequest(request_.get());
+}
+
+void DevToolsURLInterceptorRequestJob::SubRequest::Cancel() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (!fetch_in_progress_)
+    return;
+
+  fetch_in_progress_ = false;
+  request_->Cancel();
+}
+
+DevToolsURLInterceptorRequestJob::MockResponseDetails::MockResponseDetails(
+    std::string response_bytes,
+    base::TimeTicks response_time)
+    : response_bytes_(std::move(response_bytes)),
+      read_offset_(0),
+      response_time_(response_time) {
+  int header_size = net::HttpUtil::LocateEndOfHeaders(response_bytes_.c_str(),
+                                                      response_bytes_.size());
+  if (header_size == -1) {
+    LOG(WARNING) << "Can't find headers in result";
+    response_headers_ = new net::HttpResponseHeaders("");
+  } else {
+    response_headers_ =
+        new net::HttpResponseHeaders(net::HttpUtil::AssembleRawHeaders(
+            response_bytes_.c_str(), header_size));
+    read_offset_ = header_size;
+  }
+
+  CHECK_LE(read_offset_, response_bytes_.size());
+}
+
+DevToolsURLInterceptorRequestJob::MockResponseDetails::MockResponseDetails(
+    const scoped_refptr<net::HttpResponseHeaders>& response_headers,
+    std::string response_bytes,
+    size_t read_offset,
+    base::TimeTicks response_time)
+    : response_headers_(response_headers),
+      response_bytes_(std::move(response_bytes)),
+      read_offset_(read_offset),
+      response_time_(response_time) {}
+
+DevToolsURLInterceptorRequestJob::MockResponseDetails::~MockResponseDetails() {}
+
+int DevToolsURLInterceptorRequestJob::MockResponseDetails::ReadRawData(
+    net::IOBuffer* buf,
+    int buf_size) {
+  size_t bytes_available = response_bytes_.size() - read_offset_;
+  size_t bytes_to_copy =
+      std::min(static_cast<size_t>(buf_size), bytes_available);
+  if (bytes_to_copy > 0) {
+    std::memcpy(buf->data(), &response_bytes_.data()[read_offset_],
+                bytes_to_copy);
+    read_offset_ += bytes_to_copy;
+  }
+  return bytes_to_copy;
+}
+
+}  // namespace content