DevTools protocol interception, blocking & modification of requests

content/browser/devtools/protocol/network_handler.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/devtools/protocol/network_handler.h"
 
 #include <stddef.h>
 
 #include "base/barrier_closure.h"
+#include "base/base64.h"
 #include "base/command_line.h"
 #include "base/containers/hash_tables.h"
 #include "base/process/process_handle.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
 #include "content/browser/devtools/devtools_session.h"
+#include "content/browser/devtools/devtools_url_interceptor_request_job.h"
 #include "content/browser/devtools/protocol/page.h"
 #include "content/browser/devtools/protocol/security.h"
 #include "content/browser/frame_host/frame_tree_node.h"
 #include "content/browser/frame_host/render_frame_host_impl.h"
 #include "content/common/navigation_params.h"
 #include "content/common/resource_request.h"
 #include "content/common/resource_request_completion_status.h"
 #include "content/public/browser/browser_context.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/browsing_data_remover.h"
 #include "content/public/browser/content_browser_client.h"
 #include "content/public/browser/render_process_host.h"
 #include "content/public/browser/resource_context.h"
 #include "content/public/browser/site_instance.h"
 #include "content/public/browser/storage_partition.h"
 #include "content/public/browser/web_contents.h"
 #include "content/public/common/content_client.h"
 #include "content/public/common/content_switches.h"
 #include "content/public/common/resource_devtools_info.h"
 #include "content/public/common/resource_response.h"
 #include "net/base/net_errors.h"
+#include "net/base/upload_bytes_element_reader.h"
 #include "net/cookies/cookie_store.h"
 #include "net/http/http_response_headers.h"
 #include "net/url_request/url_request_context.h"
 #include "net/url_request/url_request_context_getter.h"
 
 namespace content {
 namespace protocol {
 namespace {
 
 using ProtocolCookieArray = Array<Network::Cookie>;
@@ skipping 282 matching lines @@
     case blink::kWebReferrerPolicyOriginWhenCrossOrigin:
       return Network::Request::ReferrerPolicyEnum::OriginWhenCrossOrigin;
     case blink::kWebReferrerPolicyNoReferrerWhenDowngradeOriginWhenCrossOrigin:
       return Network::Request::ReferrerPolicyEnum::
           NoReferrerWhenDowngradeOriginWhenCrossOrigin;
   }
   NOTREACHED();
   return Network::Request::ReferrerPolicyEnum::NoReferrerWhenDowngrade;
 }
 
+String referrerPolicy(net::URLRequest::ReferrerPolicy referrer_policy) {
+  switch (referrer_policy) {
+    case net::URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE:
+      return Network::Request::ReferrerPolicyEnum::NoReferrerWhenDowngrade;
+    case net::URLRequest::
+        REDUCE_REFERRER_GRANULARITY_ON_TRANSITION_CROSS_ORIGIN:
+      return Network::Request::ReferrerPolicyEnum::
+          NoReferrerWhenDowngradeOriginWhenCrossOrigin;
+    case net::URLRequest::ORIGIN_ONLY_ON_TRANSITION_CROSS_ORIGIN:
+      return Network::Request::ReferrerPolicyEnum::
+          NoReferrerWhenDowngradeOriginWhenCrossOrigin;
+    case net::URLRequest::NEVER_CLEAR_REFERRER:
+      return Network::Request::ReferrerPolicyEnum::Origin;
+    case net::URLRequest::ORIGIN:
+      return Network::Request::ReferrerPolicyEnum::Origin;
+    case net::URLRequest::NO_REFERRER:
+      return Network::Request::ReferrerPolicyEnum::NoReferrer;
+    default:
+      break;
+  }
+  NOTREACHED();
+  return Network::Request::ReferrerPolicyEnum::NoReferrerWhenDowngrade;
+}
+
 String securityState(const GURL& url, const net::CertStatus& cert_status) {
   if (!url.SchemeIsCryptographic())
     return Security::SecurityStateEnum::Neutral;
   if (net::IsCertStatusError(cert_status) &&
       !net::IsCertStatusMinorError(cert_status)) {
     return Security::SecurityStateEnum::Insecure;
   }
   return Security::SecurityStateEnum::Secure;
 }
 
+net::Error NetErrorFromString(const std::string& error) {
+  if (error == Network::ErrorReasonEnum::Failed)
+    return net::ERR_FAILED;
+  if (error == Network::ErrorReasonEnum::Aborted)
+    return net::ERR_ABORTED;
+  if (error == Network::ErrorReasonEnum::TimedOut)
+    return net::ERR_TIMED_OUT;
+  if (error == Network::ErrorReasonEnum::AccessDenied)
+    return net::ERR_ACCESS_DENIED;
+  if (error == Network::ErrorReasonEnum::ConnectionClosed)
+    return net::ERR_CONNECTION_CLOSED;
+  if (error == Network::ErrorReasonEnum::ConnectionReset)
+    return net::ERR_CONNECTION_RESET;
+  if (error == Network::ErrorReasonEnum::ConnectionRefused)
+    return net::ERR_CONNECTION_REFUSED;
+  if (error == Network::ErrorReasonEnum::ConnectionAborted)
+    return net::ERR_CONNECTION_ABORTED;
+  if (error == Network::ErrorReasonEnum::ConnectionFailed)
+    return net::ERR_CONNECTION_FAILED;
+  if (error == Network::ErrorReasonEnum::NameNotResolved)
+    return net::ERR_NAME_NOT_RESOLVED;
+  if (error == Network::ErrorReasonEnum::InternetDisconnected)
+    return net::ERR_INTERNET_DISCONNECTED;
+  if (error == Network::ErrorReasonEnum::AddressUnreachable)
+    return net::ERR_ADDRESS_UNREACHABLE;
+  return net::ERR_FAILED;
+}
+
 double timeDelta(base::TimeTicks time,
                  base::TimeTicks start,
                  double invalid_value = -1) {
   return time.is_null() ? invalid_value : (time - start).InMillisecondsF();
 }
 
 std::unique_ptr<Network::ResourceTiming> getTiming(
     const net::LoadTimingInfo& load_timing) {
   const base::TimeTicks kNullTicks;
   return Network::ResourceTiming::Create()
@@ skipping 53 matching lines @@
     }
   }
   return protocol;
 }
 
 }  // namespace
 
 NetworkHandler::NetworkHandler()
     : DevToolsDomainHandler(Network::Metainfo::domainName),
       host_(nullptr),
-      enabled_(false) {
-}
+      enabled_(false),
+      interception_enabled_(false),
+      weak_factory_(this) {}
 
 NetworkHandler::~NetworkHandler() {
 }
 
 // static
 std::vector<NetworkHandler*> NetworkHandler::ForAgentHost(
     DevToolsAgentHostImpl* host) {
   return DevToolsSession::HandlersForAgentHost<NetworkHandler>(
       host, Network::Metainfo::domainName);
 }
 
 void NetworkHandler::Wire(UberDispatcher* dispatcher) {
   frontend_.reset(new Network::Frontend(dispatcher->channel()));
   Network::Dispatcher::wire(dispatcher, this);
 }
 
 void NetworkHandler::SetRenderFrameHost(RenderFrameHostImpl* host) {
   host_ = host;
 }
 
 Response NetworkHandler::Enable(Maybe<int> max_total_size,
                                 Maybe<int> max_resource_size) {
   enabled_ = true;
   return Response::FallThrough();
 }
 
 Response NetworkHandler::Disable() {
   enabled_ = false;
   user_agent_ = std::string();
+  EnableFetchInterception(false);
   return Response::FallThrough();
 }
 
 Response NetworkHandler::ClearBrowserCache() {
   if (host_) {
     content::BrowsingDataRemover* remover =
         content::BrowserContext::GetBrowsingDataRemover(
             host_->GetSiteInstance()->GetProcess()->GetBrowserContext());
     remover->Remove(base::Time(), base::Time::Max(),
                     content::BrowsingDataRemover::DATA_TYPE_CACHE,
@@ skipping 299 matching lines @@
       request_id,
       base::TimeTicks::Now().ToInternalValue() /
           static_cast<double>(base::Time::kMicrosecondsPerSecond),
       Page::ResourceTypeEnum::Document, error_string, cancelled);
 }
 
 std::string NetworkHandler::UserAgentOverride() const {
   return enabled_ ? user_agent_ : std::string();
 }
 
+DispatchResponse NetworkHandler::EnableFetchInterception(bool enabled) {
+  if (interception_enabled_ == enabled)
+    return Response::OK();  // Nothing to do.
+
+  WebContents* web_contents = WebContents::FromRenderFrameHost(host_);
+  if (!web_contents)
+    return Response::OK();
+
+  DevToolsURLRequestInterceptor* devtools_url_request_interceptor =
+      DevToolsURLRequestInterceptor::FromBrowserContext(
+          web_contents->GetBrowserContext());
+  if (!devtools_url_request_interceptor)
+    return Response::OK();
+
+  if (enabled) {
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(
+            &DevToolsURLRequestInterceptor::State::StartInterceptingRequests,
+            devtools_url_request_interceptor->state().GetWeakPtr(),

[dgozman, 2017/05/25 21:29:36]

It's prohibited to dereference weak pointer on the thread different from it's
creation. Instead, we should post to IO immediately, retrieve interceptor on IO
and call method directly. Mark the method as "async" in protocol_config to get a
callback instead of returning synchronously.

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

An alternative is we can make the state ref counted.  Anyway that function is
now on the UI thread.

+            web_contents, weak_factory_.GetWeakPtr()));
+  } else {
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(
+            &DevToolsURLRequestInterceptor::State::StopInterceptingRequests,
+            devtools_url_request_interceptor->state().GetWeakPtr(),
+            web_contents));
+  }
+  interception_enabled_ = enabled;
+  return Response::OK();
+}
+
+namespace {
+void ProcessContinueRequestResultOnIoThread(
+    std::unique_ptr<NetworkHandler::ContinueRequestCallback> callback,
+    const DevToolsURLRequestInterceptor::CommandStatus& result) {
+  switch (result) {
+    case DevToolsURLRequestInterceptor::CommandStatus::OK:
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::BindOnce(&NetworkHandler::ContinueRequestCallback::sendSuccess,
+                         base::Owned(callback.release())));
+      break;
+    case DevToolsURLRequestInterceptor::CommandStatus::UnknownInterceptionId:
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::BindOnce(&NetworkHandler::ContinueRequestCallback::sendFailure,
+                         base::Owned(callback.release()),
+                         Response::InvalidParams("Invalid InterceptionId.")));
+      break;
+    case DevToolsURLRequestInterceptor::CommandStatus::CommandAlreadyProcessed:
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::BindOnce(
+              &NetworkHandler::ContinueRequestCallback::sendFailure,
+              base::Owned(callback.release()),
+              Response::InvalidParams("Response already processed.")));
+      break;
+  }
+}
+
+bool GetPostData(const net::URLRequest* request, std::string* post_data) {
+  if (!request->has_upload())
+    return false;
+
+  const net::UploadDataStream* stream = request->get_upload();
+  if (!stream->GetElementReaders())
+    return false;
+
+  if (stream->GetElementReaders()->size() == 0)
+    return false;
+
+  DCHECK_EQ(1u, stream->GetElementReaders()->size());

[dgozman, 2017/05/25 21:29:36]

Is this guaranteed by net?

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

Looks like no although this is the common case.  I wonder what we should do if
there's more than one, maybe concatenate them together?

I wish there was a string builder class in chromium, concatenating large strings
results in unfortunate performance.

[dgozman, 2017/05/30 21:44:27]

IIUC, std::string += operator is smart since it uses vector with 2x growth
factor inside.

[alex clarke (OOO till 29th), 2017/05/31 15:52:02]

Mmm OK that should help.  I'm just worried about potential O(n^2) nastiness if
there's a lot of element readers. I hope that's unlikely!

+  const net::UploadBytesElementReader* reader =
+      (*stream->GetElementReaders())[0]->AsBytesReader();
+  if (!reader)
+    return false;
+  *post_data = std::string(reader->bytes(), reader->length());

[dgozman, 2017/05/25 21:29:36]

I think we'll need to base64-encode this? Let's have a TODO (or comment on the
bug).

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

For reference it's simple to base 64 encode that here.  Unfortunately the other
places where Network::Request is created don't appear to base64 encode the
result.  Lets add a todo.

  base::Base64Encode(base::StringPiece(reader->bytes(), reader->length()),
                     post_data);

+  return true;
+}
+}  // namespace
+
+void NetworkHandler::ContinueRequest(
+    const std::string& interception_id,
+    Maybe<std::string> error_reason,
+    Maybe<std::string> base64_raw_response,
+    Maybe<std::string> url,
+    Maybe<std::string> method,
+    Maybe<std::string> post_data,
+    Maybe<protocol::Network::Headers> headers,
+    std::unique_ptr<ContinueRequestCallback> callback) {
+  DevToolsURLRequestInterceptor* devtools_url_request_interceptor =
+      DevToolsURLRequestInterceptor::FromBrowserContext(
+          WebContents::FromRenderFrameHost(host_)->GetBrowserContext());
+  if (!devtools_url_request_interceptor) {
+    callback->sendFailure(Response::InternalError());
+    return;
+  }
+
+  base::Optional<net::Error> error;
+  if (error_reason.isJust())
+    error = NetErrorFromString(error_reason.fromJust());
+
+  base::Optional<std::string> raw_response;
+  if (base64_raw_response.isJust()) {
+    std::string decoded;
+    if (!base::Base64Decode(base64_raw_response.fromJust(), &decoded)) {
+      callback->sendFailure(Response::InvalidParams("Invalid rawResponse."));
+      return;
+    }
+    raw_response = decoded;
+  }
+
+  BrowserThread::PostTask(
+      BrowserThread::IO, FROM_HERE,
+      base::BindOnce(
+          &DevToolsURLRequestInterceptor::State::ContinueRequest,
+          devtools_url_request_interceptor->state().GetWeakPtr(),

[dgozman, 2017/05/25 21:29:36]

Similar thing here.

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

Done.

+          interception_id,
+          base::MakeUnique<DevToolsURLRequestInterceptor::Modifications>(
+              std::move(error), std::move(raw_response), std::move(url),
+              std::move(method), std::move(post_data), std::move(headers)),
+          base::BindOnce(ProcessContinueRequestResultOnIoThread,
+                         base::Passed(std::move(callback)))));
+}
+
+// static
+std::unique_ptr<Network::Request> NetworkHandler::CreateRequestFromURLRequest(
+    const net::URLRequest* request) {
+  std::unique_ptr<DictionaryValue> headers_dict(DictionaryValue::create());
+  for (net::HttpRequestHeaders::Iterator it(request->extra_request_headers());
+       it.GetNext();) {
+    headers_dict->setString(it.name(), it.value());
+  }
+  std::unique_ptr<protocol::Network::Request> request_object =
+      Network::Request::Create()
+          .SetUrl(request->url().spec())
+          .SetMethod(request->method())
+          .SetHeaders(Object::fromValue(headers_dict.get(), nullptr))
+          .SetInitialPriority(resourcePriority(request->priority()))
+          .SetReferrerPolicy(referrerPolicy(request->referrer_policy()))
+          .Build();
+  std::string post_data;
+  if (GetPostData(request, &post_data))
+    request_object->SetPostData(std::move(post_data));
+  return request_object;
+}
+
 }  // namespace protocol
 }  // namespace content