DevTools protocol interception, blocking & modification of requests

content/browser/devtools/protocol/network_handler.cc

Index: content/browser/devtools/protocol/network_handler.cc
diff --git a/content/browser/devtools/protocol/network_handler.cc b/content/browser/devtools/protocol/network_handler.cc
index 62abd9ece33f18a6bb49276e8ef33653c28e7fee..9b5a89de111a7be67f7deeb2fad8e74f5c42e725 100644
--- a/content/browser/devtools/protocol/network_handler.cc
+++ b/content/browser/devtools/protocol/network_handler.cc
@@ -7,6 +7,7 @@
 #include <stddef.h>
 
 #include "base/barrier_closure.h"
+#include "base/base64.h"
 #include "base/command_line.h"
 #include "base/containers/hash_tables.h"
 #include "base/process/process_handle.h"
@@ -14,6 +15,7 @@
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
 #include "content/browser/devtools/devtools_session.h"
+#include "content/browser/devtools/devtools_url_interceptor_request_job.h"
 #include "content/browser/devtools/protocol/page.h"
 #include "content/browser/devtools/protocol/security.h"
 #include "content/browser/frame_host/frame_tree_node.h"
@@ -35,6 +37,7 @@
 #include "content/public/common/resource_devtools_info.h"
 #include "content/public/common/resource_response.h"
 #include "net/base/net_errors.h"
+#include "net/base/upload_bytes_element_reader.h"
 #include "net/cookies/cookie_store.h"
 #include "net/http/http_response_headers.h"
 #include "net/url_request/url_request_context.h"
@@ -337,6 +340,30 @@ String referrerPolicy(blink::WebReferrerPolicy referrer_policy) {
   return Network::Request::ReferrerPolicyEnum::NoReferrerWhenDowngrade;
 }
 
+String referrerPolicy(net::URLRequest::ReferrerPolicy referrer_policy) {
+  switch (referrer_policy) {
+    case net::URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE:
+      return Network::Request::ReferrerPolicyEnum::NoReferrerWhenDowngrade;
+    case net::URLRequest::
+        REDUCE_REFERRER_GRANULARITY_ON_TRANSITION_CROSS_ORIGIN:
+      return Network::Request::ReferrerPolicyEnum::
+          NoReferrerWhenDowngradeOriginWhenCrossOrigin;
+    case net::URLRequest::ORIGIN_ONLY_ON_TRANSITION_CROSS_ORIGIN:
+      return Network::Request::ReferrerPolicyEnum::
+          NoReferrerWhenDowngradeOriginWhenCrossOrigin;
+    case net::URLRequest::NEVER_CLEAR_REFERRER:
+      return Network::Request::ReferrerPolicyEnum::Origin;
+    case net::URLRequest::ORIGIN:
+      return Network::Request::ReferrerPolicyEnum::Origin;
+    case net::URLRequest::NO_REFERRER:
+      return Network::Request::ReferrerPolicyEnum::NoReferrer;
+    default:
+      break;
+  }
+  NOTREACHED();
+  return Network::Request::ReferrerPolicyEnum::NoReferrerWhenDowngrade;
+}
+
 String securityState(const GURL& url, const net::CertStatus& cert_status) {
   if (!url.SchemeIsCryptographic())
     return Security::SecurityStateEnum::Neutral;
@@ -347,6 +374,34 @@ String securityState(const GURL& url, const net::CertStatus& cert_status) {
   return Security::SecurityStateEnum::Secure;
 }
 
+net::Error NetErrorFromString(const std::string& error) {
+  if (error == Network::ErrorReasonEnum::Failed)
+    return net::ERR_FAILED;
+  if (error == Network::ErrorReasonEnum::Aborted)
+    return net::ERR_ABORTED;
+  if (error == Network::ErrorReasonEnum::TimedOut)
+    return net::ERR_TIMED_OUT;
+  if (error == Network::ErrorReasonEnum::AccessDenied)
+    return net::ERR_ACCESS_DENIED;
+  if (error == Network::ErrorReasonEnum::ConnectionClosed)
+    return net::ERR_CONNECTION_CLOSED;
+  if (error == Network::ErrorReasonEnum::ConnectionReset)
+    return net::ERR_CONNECTION_RESET;
+  if (error == Network::ErrorReasonEnum::ConnectionRefused)
+    return net::ERR_CONNECTION_REFUSED;
+  if (error == Network::ErrorReasonEnum::ConnectionAborted)
+    return net::ERR_CONNECTION_ABORTED;
+  if (error == Network::ErrorReasonEnum::ConnectionFailed)
+    return net::ERR_CONNECTION_FAILED;
+  if (error == Network::ErrorReasonEnum::NameNotResolved)
+    return net::ERR_NAME_NOT_RESOLVED;
+  if (error == Network::ErrorReasonEnum::InternetDisconnected)
+    return net::ERR_INTERNET_DISCONNECTED;
+  if (error == Network::ErrorReasonEnum::AddressUnreachable)
+    return net::ERR_ADDRESS_UNREACHABLE;
+  return net::ERR_FAILED;
+}
+
 double timeDelta(base::TimeTicks time,
                  base::TimeTicks start,
                  double invalid_value = -1) {
@@ -420,8 +475,9 @@ String getProtocol(const GURL& url, const ResourceResponseHead& head) {
 NetworkHandler::NetworkHandler()
     : DevToolsDomainHandler(Network::Metainfo::domainName),
       host_(nullptr),
-      enabled_(false) {
-}
+      enabled_(false),
+      interception_enabled_(false),
+      weak_factory_(this) {}
 
 NetworkHandler::~NetworkHandler() {
 }
@@ -451,6 +507,7 @@ Response NetworkHandler::Enable(Maybe<int> max_total_size,
 Response NetworkHandler::Disable() {
   enabled_ = false;
   user_agent_ = std::string();
+  EnableFetchInterception(false);
   return Response::FallThrough();
 }
 
@@ -770,5 +827,154 @@ std::string NetworkHandler::UserAgentOverride() const {
   return enabled_ ? user_agent_ : std::string();
 }
 
+DispatchResponse NetworkHandler::EnableFetchInterception(bool enabled) {
+  if (interception_enabled_ == enabled)
+    return Response::OK();  // Nothing to do.
+
+  WebContents* web_contents = WebContents::FromRenderFrameHost(host_);
+  if (!web_contents)
+    return Response::OK();
+
+  DevToolsURLRequestInterceptor* devtools_url_request_interceptor =
+      DevToolsURLRequestInterceptor::FromBrowserContext(
+          web_contents->GetBrowserContext());
+  if (!devtools_url_request_interceptor)
+    return Response::OK();
+
+  if (enabled) {
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(
+            &DevToolsURLRequestInterceptor::State::StartInterceptingRequests,
+            devtools_url_request_interceptor->state().GetWeakPtr(),

[dgozman, 2017/05/25 21:29:36]

It's prohibited to dereference weak pointer on the thread different from it's
creation. Instead, we should post to IO immediately, retrieve interceptor on IO
and call method directly. Mark the method as "async" in protocol_config to get a
callback instead of returning synchronously.

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

An alternative is we can make the state ref counted.  Anyway that function is
now on the UI thread.

+            web_contents, weak_factory_.GetWeakPtr()));
+  } else {
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(
+            &DevToolsURLRequestInterceptor::State::StopInterceptingRequests,
+            devtools_url_request_interceptor->state().GetWeakPtr(),
+            web_contents));
+  }
+  interception_enabled_ = enabled;
+  return Response::OK();
+}
+
+namespace {
+void ProcessContinueRequestResultOnIoThread(
+    std::unique_ptr<NetworkHandler::ContinueRequestCallback> callback,
+    const DevToolsURLRequestInterceptor::CommandStatus& result) {
+  switch (result) {
+    case DevToolsURLRequestInterceptor::CommandStatus::OK:
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::BindOnce(&NetworkHandler::ContinueRequestCallback::sendSuccess,
+                         base::Owned(callback.release())));
+      break;
+    case DevToolsURLRequestInterceptor::CommandStatus::UnknownInterceptionId:
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::BindOnce(&NetworkHandler::ContinueRequestCallback::sendFailure,
+                         base::Owned(callback.release()),
+                         Response::InvalidParams("Invalid InterceptionId.")));
+      break;
+    case DevToolsURLRequestInterceptor::CommandStatus::CommandAlreadyProcessed:
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::BindOnce(
+              &NetworkHandler::ContinueRequestCallback::sendFailure,
+              base::Owned(callback.release()),
+              Response::InvalidParams("Response already processed.")));
+      break;
+  }
+}
+
+bool GetPostData(const net::URLRequest* request, std::string* post_data) {
+  if (!request->has_upload())
+    return false;
+
+  const net::UploadDataStream* stream = request->get_upload();
+  if (!stream->GetElementReaders())
+    return false;
+
+  if (stream->GetElementReaders()->size() == 0)
+    return false;
+
+  DCHECK_EQ(1u, stream->GetElementReaders()->size());

[dgozman, 2017/05/25 21:29:36]

Is this guaranteed by net?

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

Looks like no although this is the common case.  I wonder what we should do if
there's more than one, maybe concatenate them together?

I wish there was a string builder class in chromium, concatenating large strings
results in unfortunate performance.

[dgozman, 2017/05/30 21:44:27]

IIUC, std::string += operator is smart since it uses vector with 2x growth
factor inside.

[alex clarke (OOO till 29th), 2017/05/31 15:52:02]

Mmm OK that should help.  I'm just worried about potential O(n^2) nastiness if
there's a lot of element readers. I hope that's unlikely!

+  const net::UploadBytesElementReader* reader =
+      (*stream->GetElementReaders())[0]->AsBytesReader();
+  if (!reader)
+    return false;
+  *post_data = std::string(reader->bytes(), reader->length());

[dgozman, 2017/05/25 21:29:36]

I think we'll need to base64-encode this? Let's have a TODO (or comment on the
bug).

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

For reference it's simple to base 64 encode that here.  Unfortunately the other
places where Network::Request is created don't appear to base64 encode the
result.  Lets add a todo.

  base::Base64Encode(base::StringPiece(reader->bytes(), reader->length()),
                     post_data);

+  return true;
+}
+}  // namespace
+
+void NetworkHandler::ContinueRequest(
+    const std::string& interception_id,
+    Maybe<std::string> error_reason,
+    Maybe<std::string> base64_raw_response,
+    Maybe<std::string> url,
+    Maybe<std::string> method,
+    Maybe<std::string> post_data,
+    Maybe<protocol::Network::Headers> headers,
+    std::unique_ptr<ContinueRequestCallback> callback) {
+  DevToolsURLRequestInterceptor* devtools_url_request_interceptor =
+      DevToolsURLRequestInterceptor::FromBrowserContext(
+          WebContents::FromRenderFrameHost(host_)->GetBrowserContext());
+  if (!devtools_url_request_interceptor) {
+    callback->sendFailure(Response::InternalError());
+    return;
+  }
+
+  base::Optional<net::Error> error;
+  if (error_reason.isJust())
+    error = NetErrorFromString(error_reason.fromJust());
+
+  base::Optional<std::string> raw_response;
+  if (base64_raw_response.isJust()) {
+    std::string decoded;
+    if (!base::Base64Decode(base64_raw_response.fromJust(), &decoded)) {
+      callback->sendFailure(Response::InvalidParams("Invalid rawResponse."));
+      return;
+    }
+    raw_response = decoded;
+  }
+
+  BrowserThread::PostTask(
+      BrowserThread::IO, FROM_HERE,
+      base::BindOnce(
+          &DevToolsURLRequestInterceptor::State::ContinueRequest,
+          devtools_url_request_interceptor->state().GetWeakPtr(),

[dgozman, 2017/05/25 21:29:36]

Similar thing here.

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

Done.

+          interception_id,
+          base::MakeUnique<DevToolsURLRequestInterceptor::Modifications>(
+              std::move(error), std::move(raw_response), std::move(url),
+              std::move(method), std::move(post_data), std::move(headers)),
+          base::BindOnce(ProcessContinueRequestResultOnIoThread,
+                         base::Passed(std::move(callback)))));
+}
+
+// static
+std::unique_ptr<Network::Request> NetworkHandler::CreateRequestFromURLRequest(
+    const net::URLRequest* request) {
+  std::unique_ptr<DictionaryValue> headers_dict(DictionaryValue::create());
+  for (net::HttpRequestHeaders::Iterator it(request->extra_request_headers());
+       it.GetNext();) {
+    headers_dict->setString(it.name(), it.value());
+  }
+  std::unique_ptr<protocol::Network::Request> request_object =
+      Network::Request::Create()
+          .SetUrl(request->url().spec())
+          .SetMethod(request->method())
+          .SetHeaders(Object::fromValue(headers_dict.get(), nullptr))
+          .SetInitialPriority(resourcePriority(request->priority()))
+          .SetReferrerPolicy(referrerPolicy(request->referrer_policy()))
+          .Build();
+  std::string post_data;
+  if (GetPostData(request, &post_data))
+    request_object->SetPostData(std::move(post_data));
+  return request_object;
+}
+
 }  // namespace protocol
 }  // namespace content