DevTools protocol interception, blocking & modification of requests

content/browser/devtools/devtools_url_interceptor_request_job.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/devtools/devtools_url_interceptor_request_job.h"
+
+#include "base/memory/ptr_util.h"
+#include "base/strings/stringprintf.h"
+#include "content/browser/devtools/protocol/network_handler.h"
+#include "net/base/elements_upload_data_stream.h"
+#include "net/base/io_buffer.h"
+#include "net/base/upload_bytes_element_reader.h"
+#include "net/base/upload_element_reader.h"
+#include "net/cert/cert_status_flags.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_util.h"
+#include "net/url_request/url_request_context.h"
+
+namespace content {
+
+using CommandStatus = DevToolsURLRequestInterceptor::CommandStatus;
+
+namespace {
+class ProxyUploadElementReader : public net::UploadElementReader {
+ public:
+  explicit ProxyUploadElementReader(net::UploadElementReader* reader)
+      : reader_(reader) {}
+
+  ~ProxyUploadElementReader() override {}
+
+  // net::UploadElementReader overrides:
+  int Init(const net::CompletionCallback& callback) override {
+    return reader_->Init(callback);
+  }
+
+  uint64_t GetContentLength() const override {
+    return reader_->GetContentLength();
+  }
+
+  uint64_t BytesRemaining() const override { return reader_->BytesRemaining(); }
+
+  bool IsInMemory() const override { return reader_->IsInMemory(); }
+
+  int Read(net::IOBuffer* buf,
+           int buf_length,
+           const net::CompletionCallback& callback) override {
+    return reader_->Read(buf, buf_length, callback);
+  }
+
+ private:
+  net::UploadElementReader* reader_;  // NOT OWNED
+
+  DISALLOW_COPY_AND_ASSIGN(ProxyUploadElementReader);
+};
+
+std::unique_ptr<net::UploadElementReader> GetUploadData(
+    net::URLRequest* request) {
+  if (!request->has_upload())
+    return nullptr;
+
+  const net::UploadDataStream* stream = request->get_upload();
+  if (!stream->GetElementReaders())
+    return nullptr;
+
+  DCHECK_EQ(1u, stream->GetElementReaders()->size());
+  return base::MakeUnique<ProxyUploadElementReader>(
+      (*stream->GetElementReaders())[0].get());
+}
+}  // namespace
+
+DevToolsURLInterceptorRequestJob::DevToolsURLInterceptorRequestJob(
+    base::WeakPtr<DevToolsURLRequestInterceptor::State>
+        devtools_url_request_interceptor_state,
+    const std::string& interception_id,
+    net::URLRequest* original_request,
+    net::NetworkDelegate* original_network_delegate,
+    base::WeakPtr<protocol::NetworkHandler> network_handler,
+    bool is_redirect)
+    : net::URLRequestJob(original_request, original_network_delegate),
+      devtools_url_request_interceptor_state_(
+          devtools_url_request_interceptor_state),
+      request_details_(original_request->url(),
+                       original_request->method(),
+                       GetUploadData(original_request),
+                       original_request->extra_request_headers(),
+                       original_request->priority(),
+                       original_request->context()),
+      waiting_for_user_response_(false),
+      interception_id_(interception_id),
+      network_handler_(network_handler),
+      io_thread_task_runner_(

[dgozman, 2017/05/25 21:29:35]

Is this used anywhere?

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

Once several patch sets ago,  Its gone now.

+          BrowserThread::GetTaskRunnerForThread(BrowserThread::IO)),
+      is_redirect_(is_redirect),
+      weak_ptr_factory_(this) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  devtools_url_request_interceptor_state_->RegisterJob(this, interception_id_);

[dgozman, 2017/05/25 21:29:35]

if () ... since it's a weak pointer?
I also think that since interceptor state creates the job, it could register
automatically.

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

Lets do the latter, we can reduce the number of methods.

+}
+
+DevToolsURLInterceptorRequestJob::~DevToolsURLInterceptorRequestJob() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (devtools_url_request_interceptor_state_) {
+    devtools_url_request_interceptor_state_->UnregisterJob(interception_id_);
+  }
+}
+
+// net::URLRequestJob implementation:
+void DevToolsURLInterceptorRequestJob::SetExtraRequestHeaders(
+    const net::HttpRequestHeaders& headers) {
+  request_details_.extra_request_headers = headers;
+}
+
+void DevToolsURLInterceptorRequestJob::Start() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (is_redirect_) {
+    // If this is a fetch in response to a redirect, we have already sent the
+    // Network.requestIntercepted event and the user opted to allow it so
+    // there's no need to send another. We can just start the SubRequest.
+    sub_request_.reset(new SubRequest(request_details_, this));
+  } else {
+    waiting_for_user_response_ = true;
+    BrowserThread::PostTask(
+        BrowserThread::UI, FROM_HERE,
+        base::Bind(&DevToolsURLInterceptorRequestJob::
+                       SendRequestInterceptedEventOnUiThread,
+                   weak_ptr_factory_.GetWeakPtr()));

[dgozman, 2017/05/25 21:29:35]

Cannot post to another thread with weak pointer created on this one.

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

Done.

+  }
+}
+
+void DevToolsURLInterceptorRequestJob::SendRequestInterceptedEventOnUiThread() {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  if (network_handler_) {
+    network_handler_->frontend()->RequestIntercepted(
+        interception_id_,
+        protocol::NetworkHandler::CreateRequestFromURLRequest(request()));

[dgozman, 2017/05/25 21:29:35]

You should call this method before posting to UI to avoid data races.

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

Done.

+  }
+}
+
+void DevToolsURLInterceptorRequestJob::Kill() {
+  if (sub_request_)
+    sub_request_->Cancel();
+
+  URLRequestJob::Kill();
+}
+
+int DevToolsURLInterceptorRequestJob::ReadRawData(net::IOBuffer* buf,
+                                                  int buf_size) {
+  if (sub_request_) {
+    int size = sub_request_->request()->Read(buf, buf_size);
+    return size;
+  } else {
+    CHECK(mock_response_details_);
+    return mock_response_details_->ReadRawData(buf, buf_size);
+  }
+}
+
+int DevToolsURLInterceptorRequestJob::GetResponseCode() const {
+  if (sub_request_) {
+    return sub_request_->request()->GetResponseCode();
+  } else {
+    CHECK(mock_response_details_);
+    return mock_response_details_->response_headers()->response_code();
+  }
+}
+
+void DevToolsURLInterceptorRequestJob::GetResponseInfo(
+    net::HttpResponseInfo* info) {
+  // NOTE this can get called during URLRequestJob::NotifyStartError in which
+  // case we might not have either a sub request or a mock response.
+  if (sub_request_) {
+    *info = sub_request_->request()->response_info();
+  } else if (mock_response_details_) {
+    info->headers = mock_response_details_->response_headers();
+  }
+}
+
+const net::HttpResponseHeaders*
+DevToolsURLInterceptorRequestJob::GetHttpResponseHeaders() const {
+  if (sub_request_) {
+    net::URLRequest* request = sub_request_->request();
+    return request->response_info().headers.get();
+  }
+  CHECK(mock_response_details_);
+  return mock_response_details_->response_headers().get();
+}
+
+bool DevToolsURLInterceptorRequestJob::GetMimeType(
+    std::string* mime_type) const {
+  const net::HttpResponseHeaders* response_headers = GetHttpResponseHeaders();
+  if (!response_headers)
+    return false;
+  return response_headers->GetMimeType(mime_type);
+}
+
+bool DevToolsURLInterceptorRequestJob::GetCharset(std::string* charset) {
+  const net::HttpResponseHeaders* response_headers = GetHttpResponseHeaders();
+  if (!response_headers)
+    return false;
+  return response_headers->GetCharset(charset);
+}
+
+void DevToolsURLInterceptorRequestJob::GetLoadTimingInfo(
+    net::LoadTimingInfo* load_timing_info) const {
+  if (sub_request_) {
+    sub_request_->request()->GetLoadTimingInfo(load_timing_info);
+  } else {
+    CHECK(mock_response_details_);
+    // Since this request is mocked most of the fields are irrelevant.
+    load_timing_info->receive_headers_end =
+        mock_response_details_->response_time();
+  }
+}
+
+void DevToolsURLInterceptorRequestJob::OnAuthRequired(
+    net::URLRequest* request,
+    net::AuthChallengeInfo* auth_info) {
+  NOTREACHED();

[dgozman, 2017/05/25 21:29:35]

Can this happen? Should we do something meaningful, like delegate to
NetworkDelegate we got when intercepting?

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

I don't think it can happen because the SubRequest does the real fetch.  I've
added a comment to explain.

+}
+
+void DevToolsURLInterceptorRequestJob::OnCertificateRequested(
+    net::URLRequest* request,
+    net::SSLCertRequestInfo* cert_request_info) {
+  NOTREACHED();
+}
+
+void DevToolsURLInterceptorRequestJob::OnSSLCertificateError(
+    net::URLRequest* request,
+    const net::SSLInfo& ssl_info,
+    bool fatal) {
+  NOTREACHED();
+}
+
+void DevToolsURLInterceptorRequestJob::OnResponseStarted(
+    net::URLRequest* request,
+    int net_error) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(sub_request_);
+  DCHECK_EQ(request, sub_request_->request());
+  DCHECK_NE(net::ERR_IO_PENDING, net_error);
+
+  if (net_error != net::OK) {
+    sub_request_->Cancel();
+
+    NotifyStartError(net::URLRequestStatus(net::URLRequestStatus::FAILED,
+                                           static_cast<net::Error>(net_error)));
+    return;
+  }
+
+  NotifyHeadersComplete();
+}
+
+void DevToolsURLInterceptorRequestJob::OnReadCompleted(net::URLRequest* request,
+                                                       int num_bytes) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK_EQ(request, sub_request_->request());
+
+  ReadRawDataComplete(num_bytes);
+}
+
+void DevToolsURLInterceptorRequestJob::OnReceivedRedirect(
+    net::URLRequest* request,
+    const net::RedirectInfo& redirectinfo,
+    bool* defer_redirect) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  *defer_redirect = true;
+
+  size_t iter = 0;
+  std::string header_name;
+  std::string header_value;
+  std::unique_ptr<protocol::DictionaryValue> headers_dict(
+      protocol::DictionaryValue::create());
+  while (request->response_headers()->EnumerateHeaderLines(&iter, &header_name,
+                                                           &header_value)) {
+    headers_dict->setString(header_name, header_value);
+  }
+
+  redirect_.reset(new net::RedirectInfo(redirectinfo));
+  sub_request_->Cancel();
+  sub_request_.reset();
+
+  waiting_for_user_response_ = true;
+  BrowserThread::PostTask(
+      BrowserThread::UI, FROM_HERE,
+      base::Bind(&DevToolsURLInterceptorRequestJob::
+                     SendRedirectInterceptedEventOnUiThread,
+                 weak_ptr_factory_.GetWeakPtr(),

[dgozman, 2017/05/25 21:29:35]

Ditto.

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

Done.

+                 base::Owned(headers_dict.release()), redirectinfo.status_code,
+                 redirectinfo.new_url.spec()));
+}
+
+void DevToolsURLInterceptorRequestJob::SendRedirectInterceptedEventOnUiThread(
+    protocol::DictionaryValue* headers_dict,
+    int http_status_code,
+    std::string redirect_url) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  return network_handler_->frontend()->RequestIntercepted(
+      interception_id_,
+      protocol::NetworkHandler::CreateRequestFromURLRequest(request()),

[dgozman, 2017/05/25 21:29:35]

Ditto.

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

Done.

+      protocol::Object::fromValue(headers_dict, nullptr), http_status_code,
+      redirect_url);
+}
+
+CommandStatus DevToolsURLInterceptorRequestJob::ContinueRequest(
+    std::unique_ptr<DevToolsURLRequestInterceptor::Modifications>
+        modifications) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (!waiting_for_user_response_)
+    return CommandStatus::CommandAlreadyProcessed;
+  waiting_for_user_response_ = false;
+
+  if (modifications->error_reason) {
+    NotifyStartError(net::URLRequestStatus(net::URLRequestStatus::FAILED,
+                                           *modifications->error_reason));
+    return CommandStatus::OK;
+  }
+
+  if (modifications->raw_response) {
+    mock_response_details_.reset(new MockResponseDetails(
+        std::move(*modifications->raw_response), base::TimeTicks::Now()));
+
+    std::string value;
+    if (mock_response_details_->response_headers()->IsRedirect(&value)) {
+      DCHECK(devtools_url_request_interceptor_state_);
+      devtools_url_request_interceptor_state_->ExpectRequestAfterRedirect(
+          request(), interception_id_);
+    }
+    NotifyHeadersComplete();
+    return CommandStatus::OK;
+  }
+
+  if (redirect_) {

[dgozman, 2017/05/25 21:29:35]

So, this is to proceed after redirect? Very nice!

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

Acknowledged.

+    // NOTE we don't append the text form of the status code because
+    // net::HttpResponseHeaders doesn't need that.
+    std::string raw_headers =
+        base::StringPrintf("HTTP/1.1 %d", redirect_->status_code);
+    raw_headers.append(1, '\0');
+    raw_headers.append("Location: ");
+    raw_headers.append(
+        modifications->modified_url.fromMaybe(redirect_->new_url.spec()));
+    raw_headers.append(2, '\0');
+    mock_response_details_.reset(new MockResponseDetails(
+        make_scoped_refptr(new net::HttpResponseHeaders(raw_headers)), "", 0,
+        base::TimeTicks::Now()));
+    redirect_.reset();
+
+    DCHECK(devtools_url_request_interceptor_state_);
+    devtools_url_request_interceptor_state_->ExpectRequestAfterRedirect(
+        request(), interception_id_);
+    NotifyHeadersComplete();
+  } else {
+    // Note this redirect is not visible to the caller by design. If they want a
+    // visible redirect they can mock a response with a 302.
+    if (modifications->modified_url.isJust())
+      request_details_.url = GURL(modifications->modified_url.fromJust());
+
+    if (modifications->modified_method.isJust())
+      request_details_.method = modifications->modified_method.fromJust();
+
+    if (modifications->modified_post_data.isJust()) {
+      const std::string& post_data =
+          modifications->modified_post_data.fromJust();
+      std::vector<char> data(post_data.begin(), post_data.end());
+      request_details_.post_data.reset(
+          new net::UploadOwnedBytesElementReader(&data));
+    }
+
+    if (modifications->modified_headers.isJust()) {
+      request_details_.extra_request_headers.Clear();
+      std::unique_ptr<protocol::DictionaryValue> headers =
+          modifications->modified_headers.fromJust()->toValue();
+      for (size_t i = 0; i < headers->size(); i++) {
+        std::string value;
+        if (headers->at(i).second->asString(&value)) {
+          request_details_.extra_request_headers.SetHeader(headers->at(i).first,
+                                                           value);
+        }
+      }
+    }
+
+    // The reason we start a sub request is because we are in full control of it
+    // and can choose to ignore it if, for example, the fetch encounters a
+    // redirect that the user chooses to replace with a mock response.
+    sub_request_.reset(new SubRequest(request_details_, this));
+  }
+  return CommandStatus::OK;
+}
+
+DevToolsURLInterceptorRequestJob::RequestDetails::RequestDetails(
+    const GURL& url,
+    const std::string& method,
+    std::unique_ptr<net::UploadElementReader> post_data,
+    const net::HttpRequestHeaders& extra_request_headers,
+    const net::RequestPriority& priority,
+    const net::URLRequestContext* url_request_context)
+    : url(url),
+      method(method),
+      post_data(std::move(post_data)),
+      extra_request_headers(extra_request_headers),
+      priority(priority),
+      url_request_context(url_request_context) {}
+
+DevToolsURLInterceptorRequestJob::RequestDetails::~RequestDetails() {}
+
+DevToolsURLInterceptorRequestJob::SubRequest::SubRequest(
+    DevToolsURLInterceptorRequestJob::RequestDetails& request_details,
+    DevToolsURLInterceptorRequestJob* devtools_interceptor_request_job)
+    : devtools_interceptor_request_job_(devtools_interceptor_request_job),
+      devtools_url_request_interceptor_state_(
+          devtools_interceptor_request_job_
+              ->devtools_url_request_interceptor_state_),
+      fetch_in_progress_(true) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  request_ = request_details.url_request_context->CreateRequest(
+      request_details.url, request_details.priority,
+      devtools_interceptor_request_job_),
+  request_->set_method(request_details.method);
+  request_->SetExtraRequestHeaders(request_details.extra_request_headers);
+
+  if (request_details.post_data) {
+    request_->set_upload(net::ElementsUploadDataStream::CreateWithReader(
+        std::move(request_details.post_data), 0));
+  }
+
+  if (!devtools_url_request_interceptor_state_)

[dgozman, 2017/05/25 21:29:35]

IIUC, this only happens on profile shutdown, so we can just return everywhere?

[alex clarke (OOO till 29th), 2017/05/26 19:37:02]

We can also make the state refcounted which guarantees correctness here.

+    return;
+  devtools_url_request_interceptor_state_->RegisterSubRequest(request_.get());
+  request_->Start();
+}
+
+DevToolsURLInterceptorRequestJob::SubRequest::~SubRequest() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  if (!devtools_url_request_interceptor_state_)
+    return;
+  devtools_url_request_interceptor_state_->UnregisterSubRequest(request_.get());
+}
+
+void DevToolsURLInterceptorRequestJob::SubRequest::Cancel() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  if (!fetch_in_progress_)
+    return;
+
+  fetch_in_progress_ = false;
+  request_->Cancel();
+}
+
+DevToolsURLInterceptorRequestJob::MockResponseDetails::MockResponseDetails(
+    std::string response_bytes,
+    base::TimeTicks response_time)
+    : response_bytes_(std::move(response_bytes)),
+      read_offset_(0),
+      response_time_(response_time) {
+  int header_size = net::HttpUtil::LocateEndOfHeaders(response_bytes_.c_str(),
+                                                      response_bytes_.size());
+  if (header_size == -1) {
+    LOG(WARNING) << "Can't find headers in result";
+    response_headers_ = new net::HttpResponseHeaders("");
+  } else {
+    response_headers_ =
+        new net::HttpResponseHeaders(net::HttpUtil::AssembleRawHeaders(
+            response_bytes_.c_str(), header_size));
+    read_offset_ = header_size;
+  }
+
+  CHECK_LE(read_offset_, response_bytes_.size());
+}
+
+DevToolsURLInterceptorRequestJob::MockResponseDetails::MockResponseDetails(
+    const scoped_refptr<net::HttpResponseHeaders>& response_headers,
+    std::string response_bytes,
+    size_t read_offset,
+    base::TimeTicks response_time)
+    : response_headers_(response_headers),
+      response_bytes_(std::move(response_bytes)),
+      read_offset_(read_offset),
+      response_time_(response_time) {}
+
+DevToolsURLInterceptorRequestJob::MockResponseDetails::~MockResponseDetails() {}
+
+int DevToolsURLInterceptorRequestJob::MockResponseDetails::ReadRawData(
+    net::IOBuffer* buf,
+    int buf_size) {
+  size_t bytes_available = response_bytes_.size() - read_offset_;
+  size_t bytes_to_copy =
+      std::min(static_cast<size_t>(buf_size), bytes_available);
+  if (bytes_to_copy > 0) {
+    std::memcpy(buf->data(), &response_bytes_.data()[read_offset_],
+                bytes_to_copy);
+    read_offset_ += bytes_to_copy;
+  }
+  return bytes_to_copy;
+}
+
+}  // namespace content