DevTools protocol interception, blocking & modification of requests

content/browser/devtools/devtools_url_request_interceptor.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/devtools/devtools_url_request_interceptor.h"
+
+#include "base/memory/ptr_util.h"
+#include "base/strings/stringprintf.h"
+#include "base/supports_user_data.h"
+#include "content/browser/devtools/devtools_agent_host_impl.h"
+#include "content/browser/devtools/devtools_url_interceptor_request_job.h"
+#include "content/browser/devtools/protocol/network_handler.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/devtools_agent_host.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/resource_request_info.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/browser/web_contents_observer.h"
+#include "net/http/http_request_headers.h"
+#include "net/url_request/url_request.h"
+
+namespace content {
+
+namespace {
+const char kDevToolsURLRequestInterceptorKeyName[] =
+    "DevToolsURLRequestInterceptor";
+
+class DevToolsURLRequestInterceptorUserData
+    : public base::SupportsUserData::Data {
+ public:
+  explicit DevToolsURLRequestInterceptorUserData(
+      DevToolsURLRequestInterceptor* devtools_url_request_interceptor)
+      : devtools_url_request_interceptor_(devtools_url_request_interceptor) {}
+
+  DevToolsURLRequestInterceptor* devtools_url_request_interceptor() const {
+    return devtools_url_request_interceptor_;
+  }
+
+ private:
+  DevToolsURLRequestInterceptor* devtools_url_request_interceptor_;
+
+  DISALLOW_COPY_AND_ASSIGN(DevToolsURLRequestInterceptorUserData);
+};
+
+}  // namespace
+
+DevToolsURLRequestInterceptor::DevToolsURLRequestInterceptor(
+    BrowserContext* browser_context)
+    : browser_context_(browser_context), state_(new State()) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  browser_context_->SetUserData(
+      kDevToolsURLRequestInterceptorKeyName,
+      base::MakeUnique<DevToolsURLRequestInterceptorUserData>(this));
+}
+
+DevToolsURLRequestInterceptor::~DevToolsURLRequestInterceptor() {
+  // The BrowserContext owns us, so we don't need to unregister
+  // DevToolsURLRequestInterceptorUserData explicitly.
+}
+
+net::URLRequestJob* DevToolsURLRequestInterceptor::MaybeInterceptRequest(
+    net::URLRequest* request,
+    net::NetworkDelegate* network_delegate) const {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  base::WeakPtr<protocol::NetworkHandler> network_handler;
+  if (!state().ShouldInterceptRequest(request, &network_handler))
+    return nullptr;
+
+  bool is_redirect;
+  std::string intercept_id = state().GetIdForRequest(request, &is_redirect);
+  return new DevToolsURLInterceptorRequestJob(
+      state().GetWeakPtr(), intercept_id, request, network_delegate,
+      network_handler, is_redirect);
+}
+
+net::URLRequestJob* DevToolsURLRequestInterceptor::MaybeInterceptRedirect(
+    net::URLRequest* request,
+    net::NetworkDelegate* network_delegate,
+    const GURL& location) const {
+  return nullptr;
+}
+
+net::URLRequestJob* DevToolsURLRequestInterceptor::MaybeInterceptResponse(
+    net::URLRequest* request,
+    net::NetworkDelegate* network_delegate) const {
+  return nullptr;
+}
+
+DevToolsURLRequestInterceptor::State::State()
+    : next_id_(0), weak_ptr_factory_(this) {}
+
+DevToolsURLRequestInterceptor::State::~State() {}
+
+void DevToolsURLRequestInterceptor::State::AllowRequest(
+    std::string interception_id,
+    CommandCallback callback) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DevToolsURLInterceptorRequestJob* job = GetJob(interception_id);
+  if (!job) {
+    std::move(callback).Run(CommandStatus::UnknownInterceptionId);
+  } else {
+    std::move(callback).Run(job->AllowRequest());
+  }
+}
+
+void DevToolsURLRequestInterceptor::State::BlockRequest(
+    std::string interception_id,
+    net::Error error_reason,
+    CommandCallback callback) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DevToolsURLInterceptorRequestJob* job = GetJob(interception_id);
+  if (!job) {
+    std::move(callback).Run(CommandStatus::UnknownInterceptionId);
+  } else {
+    std::move(callback).Run(job->BlockRequest(error_reason));
+  }
+}
+
+void DevToolsURLRequestInterceptor::State::ModifyRequest(
+    std::string interception_id,
+    std::unique_ptr<Modifications> modifications,
+    CommandCallback callback) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DevToolsURLInterceptorRequestJob* job = GetJob(interception_id);
+  if (!job) {
+    std::move(callback).Run(CommandStatus::UnknownInterceptionId);
+  } else {
+    std::move(callback).Run(job->ModifyRequest(std::move(modifications)));
+  }
+}
+
+void DevToolsURLRequestInterceptor::State::MockResponse(
+    std::string interception_id,
+    std::string raw_response,
+    CommandCallback callback) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DevToolsURLInterceptorRequestJob* job = GetJob(interception_id);
+  if (!job) {
+    std::move(callback).Run(CommandStatus::UnknownInterceptionId);
+  } else {
+    std::move(callback).Run(job->MockResponse(std::move(raw_response)));
+  }
+}
+
+bool DevToolsURLRequestInterceptor::State::ShouldInterceptRequest(
+    const net::URLRequest* request,
+    base::WeakPtr<protocol::NetworkHandler>* network_handler) const {
+  const ResourceRequestInfo* resource_request_info =
+      ResourceRequestInfo::ForRequest(request);
+  if (!resource_request_info)
+    return false;
+  int frame_tree_node_id = resource_request_info->GetFrameTreeNodeId();
+  if (frame_tree_node_id == -1) {
+    // |frame_tree_node_id| is not set for renderer side requests, fall back to
+    // the RenderFrameID.
+    int render_frame_id = resource_request_info->GetRenderFrameID();

[dgozman, 2017/05/22 22:27:04]

Note that render frame ids clash between different processes. How do you handle
it? Should we use GetFrameTreeNodeIdGetterForRequest instead?

[alex clarke (OOO till 29th), 2017/05/23 15:46:56]

There was a bug here, it needed the process id too in the map key.

Ideally I'd use GetFrameTreeNodeIdGetterForRequest or
GetWebContentsGetterForRequest but both of those are UI thread only.

+    const auto find_it = intercepted_render_frames_.find(render_frame_id);
+    if (find_it == intercepted_render_frames_.end())
+      return false;
+    *network_handler = find_it->second;
+  } else {
+    // |frame_tree_node_id| is set for browser side navigations, so use that
+    // because the RenderFrameID isn't known.
+    const auto find_it = intercepted_frame_tree_nodes_.find(frame_tree_node_id);
+    if (find_it == intercepted_frame_tree_nodes_.end())
+      return false;
+    *network_handler = find_it->second;
+  }
+
+  // We don't want to intercept our own sub requests.
+  return sub_requests_.find(request) == sub_requests_.end();
+}
+
+class DevToolsURLRequestInterceptor::State::InterceptedWebContentsObserver
+    : public WebContentsObserver {
+ public:
+  InterceptedWebContentsObserver(
+      WebContents* web_contents,
+      base::WeakPtr<DevToolsURLRequestInterceptor::State> state,
+      base::WeakPtr<protocol::NetworkHandler> network_handler)
+      : WebContentsObserver(web_contents),
+        state_(state),
+        network_handler_(network_handler) {}
+
+  void RenderFrameCreated(RenderFrameHost* render_frame_host) override {
+    DCHECK_CURRENTLY_ON(BrowserThread::UI);
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(&DevToolsURLRequestInterceptor::State::
+                           StartInterceptingRequestsInternal,
+                       state_, render_frame_host->GetRoutingID(),
+                       render_frame_host->GetFrameTreeNodeId(),
+                       network_handler_));
+  }
+
+  void RenderFrameDeleted(RenderFrameHost* render_frame_host) override {
+    DCHECK_CURRENTLY_ON(BrowserThread::UI);
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(&DevToolsURLRequestInterceptor::State::
+                           StopInterceptingRequestsInternal,
+                       state_, render_frame_host->GetRoutingID(),
+                       render_frame_host->GetFrameTreeNodeId()));
+  }
+
+ private:
+  base::WeakPtr<DevToolsURLRequestInterceptor::State> state_;
+  base::WeakPtr<protocol::NetworkHandler> network_handler_;
+};
+
+void DevToolsURLRequestInterceptor::State::StartInterceptingRequestsInternal(
+    int render_frame_id,
+    int frame_tree_node_id,
+    base::WeakPtr<protocol::NetworkHandler> network_handler) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  intercepted_render_frames_[render_frame_id] = network_handler;
+  intercepted_frame_tree_nodes_[frame_tree_node_id] = network_handler;

[dgozman, 2017/05/22 22:27:04]

It's unfortunate that you have to observe and keep track of these yourself. This
does not work with OOPIF for example, since we have complex logic which
determines NetworkHandler->RFHI mapping.

Can we retrieve NetworkHandler through RenderFrameDevToolsAgentHost instead?

[alex clarke (OOO till 29th), 2017/05/23 15:46:56]

Agreed, I'd rather not do that but we need to do the look up on the IO thread
which complicates matters.
It seems to work, I visited several ad-encrusted sites with --site-per-process
enabled and I saw interception events for ad iframes.  Note multiple processes
can map to the same NetworkHandler.
Not sure, but it would need to work from the IO thread.

+}
+
+void DevToolsURLRequestInterceptor::State::StopInterceptingRequestsInternal(
+    int render_frame_id,
+    int frame_tree_node_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  intercepted_render_frames_.erase(render_frame_id);
+  intercepted_frame_tree_nodes_.erase(frame_tree_node_id);
+}
+
+void DevToolsURLRequestInterceptor::State::StartInterceptingRequests(
+    WebContents* web_contents,
+    base::WeakPtr<protocol::NetworkHandler> network_handler) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  for (RenderFrameHost* render_frame_host : web_contents->GetAllFrames()) {
+    intercepted_render_frames_[render_frame_host->GetRoutingID()] =
+        network_handler;
+    intercepted_frame_tree_nodes_[render_frame_host->GetFrameTreeNodeId()] =
+        network_handler;
+  }
+
+  // Listen for future updates.
+  observers_.emplace(
+      web_contents,
+      base::MakeUnique<InterceptedWebContentsObserver>(
+          web_contents, weak_ptr_factory_.GetWeakPtr(), network_handler));
+}
+
+void DevToolsURLRequestInterceptor::State::StopInterceptingRequests(
+    WebContents* web_contents) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  for (RenderFrameHost* render_frame_host : web_contents->GetAllFrames()) {
+    intercepted_render_frames_.erase(render_frame_host->GetRoutingID());
+    intercepted_frame_tree_nodes_.erase(
+        render_frame_host->GetFrameTreeNodeId());
+  }
+  observers_.erase(web_contents);
+}
+
+void DevToolsURLRequestInterceptor::State::RegisterSubRequest(
+    const net::URLRequest* sub_request) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(sub_requests_.find(sub_request) == sub_requests_.end());
+  sub_requests_.insert(sub_request);
+}
+
+void DevToolsURLRequestInterceptor::State::UnregisterSubRequest(
+    const net::URLRequest* sub_request) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(sub_requests_.find(sub_request) != sub_requests_.end());
+  sub_requests_.erase(sub_request);
+}
+
+void DevToolsURLRequestInterceptor::State::ExpectRequestAfterRedirect(
+    const net::URLRequest* request,
+    std::string id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  expected_redirects_[request] = id;
+}
+
+std::string DevToolsURLRequestInterceptor::State::GetIdForRequest(
+    const net::URLRequest* request,
+    bool* is_redirect) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  auto find_it = expected_redirects_.find(request);
+  if (find_it == expected_redirects_.end()) {
+    *is_redirect = false;
+    return base::StringPrintf("id-%zu", ++next_id_);
+  }
+  *is_redirect = true;
+  std::string id = find_it->second;
+  expected_redirects_.erase(find_it);
+  return id;
+}
+
+DevToolsURLInterceptorRequestJob* DevToolsURLRequestInterceptor::State::GetJob(
+    const std::string& interception_id) const {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  const auto it = interception_id_to_job_map_.find(interception_id);
+  if (it == interception_id_to_job_map_.end())
+    return nullptr;
+  return it->second;
+}
+
+void DevToolsURLRequestInterceptor::State::RegisterJob(
+    DevToolsURLInterceptorRequestJob* job,
+    const std::string& interception_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  interception_id_to_job_map_[interception_id] = job;
+}
+
+void DevToolsURLRequestInterceptor::State::UnregisterJob(
+    const std::string& interception_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  interception_id_to_job_map_.erase(interception_id);
+}
+
+// static
+DevToolsURLRequestInterceptor*
+DevToolsURLRequestInterceptor::FromBrowserContext(BrowserContext* context) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  return static_cast<DevToolsURLRequestInterceptorUserData*>(
+             context->GetUserData(kDevToolsURLRequestInterceptorKeyName))
+      ->devtools_url_request_interceptor();
+}
+
+DevToolsURLRequestInterceptor::Modifications::Modifications(
+    protocol::Maybe<std::string> url,
+    protocol::Maybe<std::string> method,
+    protocol::Maybe<std::string> post_data,
+    protocol::Maybe<protocol::Network::Headers> headers)
+    : url(std::move(url)),
+      method(std::move(method)),
+      post_data(std::move(post_data)),
+      headers(std::move(headers)) {}
+
+DevToolsURLRequestInterceptor::Modifications::~Modifications() {}
+
+}  // namespace content