DevTools protocol interception, blocking & modification of requests

content/browser/devtools/devtools_url_request_interceptor.cc

Index: content/browser/devtools/devtools_url_request_interceptor.cc
diff --git a/content/browser/devtools/devtools_url_request_interceptor.cc b/content/browser/devtools/devtools_url_request_interceptor.cc
new file mode 100644
index 0000000000000000000000000000000000000000..76df5c252f4c232f8628fd032ffa925a8ee56e88
--- /dev/null
+++ b/content/browser/devtools/devtools_url_request_interceptor.cc
@@ -0,0 +1,312 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/devtools/devtools_url_request_interceptor.h"
+
+#include "base/memory/ptr_util.h"
+#include "base/strings/stringprintf.h"
+#include "base/supports_user_data.h"
+#include "content/browser/devtools/devtools_agent_host_impl.h"
+#include "content/browser/devtools/devtools_url_interceptor_request_job.h"
+#include "content/browser/devtools/protocol/network_handler.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/devtools_agent_host.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/render_process_host.h"
+#include "content/public/browser/resource_request_info.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/browser/web_contents_observer.h"
+#include "net/http/http_request_headers.h"
+#include "net/url_request/url_request.h"
+
+namespace content {
+
+namespace {
+const char kDevToolsURLRequestInterceptorKeyName[] =
+    "DevToolsURLRequestInterceptor";
+
+class DevToolsURLRequestInterceptorUserData
+    : public base::SupportsUserData::Data {
+ public:
+  explicit DevToolsURLRequestInterceptorUserData(
+      DevToolsURLRequestInterceptor* devtools_url_request_interceptor)
+      : devtools_url_request_interceptor_(devtools_url_request_interceptor) {}
+
+  DevToolsURLRequestInterceptor* devtools_url_request_interceptor() const {
+    return devtools_url_request_interceptor_;
+  }
+
+ private:
+  DevToolsURLRequestInterceptor* devtools_url_request_interceptor_;
+
+  DISALLOW_COPY_AND_ASSIGN(DevToolsURLRequestInterceptorUserData);
+};
+
+}  // namespace
+
+DevToolsURLRequestInterceptor::DevToolsURLRequestInterceptor(
+    BrowserContext* browser_context)
+    : browser_context_(browser_context), state_(new State()) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  browser_context_->SetUserData(
+      kDevToolsURLRequestInterceptorKeyName,
+      base::MakeUnique<DevToolsURLRequestInterceptorUserData>(this));
+}
+
+DevToolsURLRequestInterceptor::~DevToolsURLRequestInterceptor() {
+  // The BrowserContext owns us, so we don't need to unregister
+  // DevToolsURLRequestInterceptorUserData explicitly.
+}
+
+net::URLRequestJob* DevToolsURLRequestInterceptor::MaybeInterceptRequest(
+    net::URLRequest* request,
+    net::NetworkDelegate* network_delegate) const {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  base::WeakPtr<protocol::NetworkHandler> network_handler;
+  if (!state().ShouldInterceptRequest(request, &network_handler))
+    return nullptr;
+
+  bool is_redirect;
+  std::string interception_id = state().GetIdForRequest(request, &is_redirect);
+  return new DevToolsURLInterceptorRequestJob(
+      state().GetWeakPtr(), interception_id, request, network_delegate,
+      network_handler, is_redirect);
+}
+
+net::URLRequestJob* DevToolsURLRequestInterceptor::MaybeInterceptRedirect(
+    net::URLRequest* request,
+    net::NetworkDelegate* network_delegate,
+    const GURL& location) const {
+  return nullptr;
+}
+
+net::URLRequestJob* DevToolsURLRequestInterceptor::MaybeInterceptResponse(
+    net::URLRequest* request,
+    net::NetworkDelegate* network_delegate) const {
+  return nullptr;
+}
+
+DevToolsURLRequestInterceptor::State::State()
+    : next_id_(0), weak_ptr_factory_(this) {}
+
+DevToolsURLRequestInterceptor::State::~State() {}
+
+void DevToolsURLRequestInterceptor::State::ContinueRequest(
+    std::string interception_id,
+    std::unique_ptr<Modifications> modifications,
+    CommandCallback callback) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DevToolsURLInterceptorRequestJob* job = GetJob(interception_id);
+  if (!job) {
+    std::move(callback).Run(CommandStatus::UnknownInterceptionId);

[dgozman, 2017/05/25 21:29:35]

Wow, that's an interesting C++ construct.

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

Agreed, I wish it wasn't necessary.

+  } else {
+    std::move(callback).Run(job->ContinueRequest(std::move(modifications)));
+  }
+}
+
+bool DevToolsURLRequestInterceptor::State::ShouldInterceptRequest(
+    const net::URLRequest* request,
+    base::WeakPtr<protocol::NetworkHandler>* network_handler) const {
+  const ResourceRequestInfo* resource_request_info =
+      ResourceRequestInfo::ForRequest(request);
+  if (!resource_request_info)
+    return false;
+  int child_id = resource_request_info->GetChildID();
+  int frame_tree_node_id = resource_request_info->GetFrameTreeNodeId();
+  if (frame_tree_node_id == -1) {
+    // |frame_tree_node_id| is not set for renderer side requests, fall back to
+    // the RenderFrameID.
+    int render_frame_id = resource_request_info->GetRenderFrameID();
+    const auto find_it = intercepted_render_frames_.find(
+        std::make_pair(render_frame_id, child_id));
+    if (find_it == intercepted_render_frames_.end())
+      return false;
+    *network_handler = find_it->second;
+  } else {
+    // |frame_tree_node_id| is set for browser side navigations, so use that
+    // because the RenderFrameID isn't known (neither is the ChildIDe

[dgozman, 2017/05/25 21:29:36]

nit: comment looks unfinished

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

Done.

+    const auto find_it = intercepted_frame_tree_nodes_.find(frame_tree_node_id);
+    if (find_it == intercepted_frame_tree_nodes_.end())
+      return false;
+    *network_handler = find_it->second;
+  }
+
+  // We don't want to intercept our own sub requests.
+  return sub_requests_.find(request) == sub_requests_.end();

[dgozman, 2017/05/25 21:29:35]

Let's return |false| early from this method, as it's run for every single
request in chrome.

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

I agree with the sentiment of failing fast, however the sub_request_ check
should only exit if the webcontents is one we've turned interception on for so I
think this is in the right place.

One thing I haven't done, and probably should is put a fast path for quitting in
the common case where we don't want to intercept anything.

[dgozman, 2017/05/30 21:44:27]

Does this mean we have sub requests for webcontents we are not intercepting?
How's that possible?
Good idea.

[alex clarke (OOO till 29th), 2017/05/31 15:52:02]

It's not :)

There is a 1:1 relationship between a DevToolsURLInterceptorRequestJob and the
actual sub request used to perform the network fetch.  I implemented it this way
for a number of reasons:

1. To let one interceptor work across all protocols.
2. Because we want to be careful what the net::NetworkDelegate associated with
the intercepted request sees - e.g.if  we want to be able to convert a redirect
into a mocked resource, the net::NetworkDelegate must not see the redirect.

+}
+
+class DevToolsURLRequestInterceptor::State::InterceptedWebContentsObserver
+    : public WebContentsObserver {
+ public:
+  InterceptedWebContentsObserver(
+      WebContents* web_contents,
+      base::WeakPtr<DevToolsURLRequestInterceptor::State> state,
+      base::WeakPtr<protocol::NetworkHandler> network_handler)
+      : WebContentsObserver(web_contents),
+        state_(state),
+        network_handler_(network_handler) {}
+
+  void RenderFrameCreated(RenderFrameHost* render_frame_host) override {
+    DCHECK_CURRENTLY_ON(BrowserThread::UI);
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(&DevToolsURLRequestInterceptor::State::
+                           StartInterceptingRequestsInternal,
+                       state_, render_frame_host->GetRoutingID(),
+                       render_frame_host->GetFrameTreeNodeId(),
+                       render_frame_host->GetProcess()->GetID(),
+                       network_handler_));
+  }
+
+  void RenderFrameDeleted(RenderFrameHost* render_frame_host) override {
+    DCHECK_CURRENTLY_ON(BrowserThread::UI);
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(&DevToolsURLRequestInterceptor::State::
+                           StopInterceptingRequestsInternal,
+                       state_, render_frame_host->GetRoutingID(),
+                       render_frame_host->GetFrameTreeNodeId(),
+                       render_frame_host->GetProcess()->GetID()));
+  }
+
+ private:
+  base::WeakPtr<DevToolsURLRequestInterceptor::State> state_;
+  base::WeakPtr<protocol::NetworkHandler> network_handler_;
+};
+
+void DevToolsURLRequestInterceptor::State::StartInterceptingRequestsInternal(
+    int render_frame_id,
+    int frame_tree_node_id,
+    int process_id,
+    base::WeakPtr<protocol::NetworkHandler> network_handler) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  intercepted_render_frames_[std::make_pair(render_frame_id, process_id)] =
+      network_handler;
+  intercepted_frame_tree_nodes_[frame_tree_node_id] = network_handler;
+}
+
+void DevToolsURLRequestInterceptor::State::StopInterceptingRequestsInternal(
+    int render_frame_id,
+    int frame_tree_node_id,
+    int process_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  intercepted_render_frames_.erase(std::make_pair(render_frame_id, process_id));
+  intercepted_frame_tree_nodes_.erase(frame_tree_node_id);
+}
+
+void DevToolsURLRequestInterceptor::State::StartInterceptingRequests(
+    WebContents* web_contents,
+    base::WeakPtr<protocol::NetworkHandler> network_handler) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  for (RenderFrameHost* render_frame_host : web_contents->GetAllFrames()) {

[dgozman, 2017/05/25 21:29:35]

I don't think WebContents' methods are callable on IO thread. Same goes for
instantiation of WebContentsObserver. Let's instead do all of this on UI, and
post task per each frame (unfortunately) to register it's ids.

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

OK done. Too bad there are not more dchecks on the WebContents methods.

+    StartInterceptingRequestsInternal(render_frame_host->GetRoutingID(),
+                                      render_frame_host->GetFrameTreeNodeId(),
+                                      render_frame_host->GetProcess()->GetID(),
+                                      network_handler);
+  }
+
+  // Listen for future updates.
+  observers_.emplace(
+      web_contents,
+      base::MakeUnique<InterceptedWebContentsObserver>(
+          web_contents, weak_ptr_factory_.GetWeakPtr(), network_handler));
+}
+
+void DevToolsURLRequestInterceptor::State::StopInterceptingRequests(

[dgozman, 2017/05/25 21:29:35]

We should also release all requests suspended at this point to continue as is.

[alex clarke (OOO till 29th), 2017/05/26 19:37:03]

Good point, done.

+    WebContents* web_contents) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  for (RenderFrameHost* render_frame_host : web_contents->GetAllFrames()) {
+    StopInterceptingRequestsInternal(render_frame_host->GetRoutingID(),
+                                     render_frame_host->GetFrameTreeNodeId(),
+                                     render_frame_host->GetProcess()->GetID());
+  }
+  observers_.erase(web_contents);
+}
+
+void DevToolsURLRequestInterceptor::State::RegisterSubRequest(
+    const net::URLRequest* sub_request) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(sub_requests_.find(sub_request) == sub_requests_.end());
+  sub_requests_.insert(sub_request);
+}
+
+void DevToolsURLRequestInterceptor::State::UnregisterSubRequest(
+    const net::URLRequest* sub_request) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(sub_requests_.find(sub_request) != sub_requests_.end());
+  sub_requests_.erase(sub_request);
+}
+
+void DevToolsURLRequestInterceptor::State::ExpectRequestAfterRedirect(
+    const net::URLRequest* request,
+    std::string id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  expected_redirects_[request] = id;
+}
+
+std::string DevToolsURLRequestInterceptor::State::GetIdForRequest(
+    const net::URLRequest* request,
+    bool* is_redirect) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  auto find_it = expected_redirects_.find(request);
+  if (find_it == expected_redirects_.end()) {
+    *is_redirect = false;
+    return base::StringPrintf("id-%zu", ++next_id_);
+  }
+  *is_redirect = true;
+  std::string id = find_it->second;
+  expected_redirects_.erase(find_it);
+  return id;
+}
+
+DevToolsURLInterceptorRequestJob* DevToolsURLRequestInterceptor::State::GetJob(
+    const std::string& interception_id) const {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  const auto it = interception_id_to_job_map_.find(interception_id);
+  if (it == interception_id_to_job_map_.end())
+    return nullptr;
+  return it->second;
+}
+
+void DevToolsURLRequestInterceptor::State::RegisterJob(
+    DevToolsURLInterceptorRequestJob* job,
+    const std::string& interception_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  interception_id_to_job_map_[interception_id] = job;
+}
+
+void DevToolsURLRequestInterceptor::State::UnregisterJob(
+    const std::string& interception_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  interception_id_to_job_map_.erase(interception_id);
+}
+
+// static
+DevToolsURLRequestInterceptor*
+DevToolsURLRequestInterceptor::FromBrowserContext(BrowserContext* context) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  return static_cast<DevToolsURLRequestInterceptorUserData*>(
+             context->GetUserData(kDevToolsURLRequestInterceptorKeyName))
+      ->devtools_url_request_interceptor();
+}
+
+DevToolsURLRequestInterceptor::Modifications::Modifications(
+    base::Optional<net::Error> error_reason,
+    base::Optional<std::string> raw_response,
+    protocol::Maybe<std::string> modified_url,
+    protocol::Maybe<std::string> modified_method,
+    protocol::Maybe<std::string> modified_post_data,
+    protocol::Maybe<protocol::Network::Headers> modified_headers)
+    : error_reason(std::move(error_reason)),
+      raw_response(std::move(raw_response)),
+      modified_url(std::move(modified_url)),
+      modified_method(std::move(modified_method)),
+      modified_post_data(std::move(modified_post_data)),
+      modified_headers(std::move(modified_headers)) {}
+
+DevToolsURLRequestInterceptor::Modifications::~Modifications() {}
+
+}  // namespace content