Handle webuis when using the network service.

content/browser/webui/web_ui_url_loader_factory.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/webui/web_ui_url_loader_factory.h"
+
+#include <map>
+
+#include "base/bind.h"
+#include "base/lazy_instance.h"
+#include "base/logging.h"
+#include "base/memory/ref_counted_memory.h"
+#include "base/strings/string_piece.h"
+#include "base/sys_byteorder.h"
+#include "content/browser/frame_host/frame_tree_node.h"
+#include "content/browser/frame_host/render_frame_host_impl.h"
+#include "content/browser/resource_context_impl.h"
+#include "content/browser/web_contents/web_contents_impl.h"
+#include "content/browser/webui/url_data_manager_backend.h"
+#include "content/browser/webui/url_data_source_impl.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/render_process_host.h"
+#include "mojo/public/cpp/bindings/binding_set.h"
+#include "net/base/io_buffer.h"

[mmenke, 2017/05/05 19:34:15]

Is this used?

[jam, 2017/05/05 22:19:39]

nope, removed

+#include "third_party/zlib/zlib.h"
+#include "ui/base/template_expressions.h"
+
+namespace content {
+
+namespace {
+class WebUIURLLoaderFactory;
+base::LazyInstance<std::map<int, std::unique_ptr<WebUIURLLoaderFactory>>>::Leaky
+    g_factories = LAZY_INSTANCE_INITIALIZER;
+
+WebContents* GetWebContentsFromFTNID(int frame_tree_node_id) {

[scottmg, 2017/05/05 19:27:42]

Maybe you could move this function somewhere accessible, since we have it here,
in c/b/loader, and I had to add it in my CL to NavigationURLLoaderNetworkService
too.

[jam, 2017/05/05 22:19:40]

sure, reused WebContents::FromFrameTreeNodeId

+  FrameTreeNode* ftn = FrameTreeNode::GloballyFindByID(frame_tree_node_id);
+  return ftn ? WebContentsImpl::FromFrameTreeNode(ftn) : nullptr;
+}
+
+class URLLoaderImpl : public mojom::URLLoader {
+ public:
+  static void Create(mojom::URLLoaderAssociatedRequest loader,
+                     const ResourceRequest& request,
+                     int frame_tree_node_id,
+                     mojo::InterfacePtrInfo<mojom::URLLoaderClient> client_info,
+                     ResourceContext* resource_context) {
+    mojom::URLLoaderClientPtr client;
+    client.Bind(std::move(client_info));
+    new URLLoaderImpl(std::move(loader), request, frame_tree_node_id,
+                      std::move(client), resource_context);
+  }
+
+ private:
+  URLLoaderImpl(mojom::URLLoaderAssociatedRequest loader,
+                const ResourceRequest& request,
+                int frame_tree_node_id,
+                mojom::URLLoaderClientPtr client,
+                ResourceContext* resource_context)
+      : binding_(this, std::move(loader)),
+        client_(std::move(client)),
+        weak_factory_(this) {
+    // NOTE: This duplicates code in URLDataManagerBackend::StartRequest

[yzshen1, 2017/05/05 19:54:07]

In this case do we also want to add a comment in
URLDataManagerBackend::StartRequest, so that people making changes to it also
aware of this file?

[jam, 2017/05/05 22:19:40]

Done.

+    if (!URLDataManagerBackend::CheckURLIsValid(request.url)) {
+      OnError(net::ERR_INVALID_URL);
+      return;
+    }
+
+    URLDataSourceImpl* source =
+        GetURLDataManagerForResourceContext(resource_context)
+            ->GetDataSourceFromURL(request.url);
+    if (!source) {
+      OnError(net::ERR_INVALID_URL);
+      return;
+    }
+
+    if (!source->source()->ShouldServiceRequest(request.url, resource_context,
+                                                -1)) {
+      OnError(net::ERR_INVALID_URL);
+      return;
+    }
+
+    std::string path;
+    URLDataManagerBackend::URLToRequestPath(request.url, &path);
+    gzipped_ = source->source()->IsGzipped(path);
+    replacements_ = source->GetReplacements();
+
+    net::HttpRequestHeaders request_headers;
+    request_headers.AddHeadersFromString(request.headers);
+    std::string origin_header;
+    request_headers.GetHeader(net::HttpRequestHeaders::kOrigin, &origin_header);
+
+    scoped_refptr<net::HttpResponseHeaders> headers =
+        URLDataManagerBackend::GetHeaders(source, path, origin_header);
+
+    ResourceResponseHead head;

[scottmg, 2017/05/05 19:27:42]

(This is more a general question, not specific to this CL)

Do you have any thoughts on handling creation of these responses? There's a lot
of fields in ResourceResponseInfo, and I'm not too clear on what's important
when. I had to fabricate one of these for the nav request for sw so I just
filled in a few fields that seemed easy to do, but it seems a bit hokey.

[jam, 2017/05/05 22:19:40]

I agree, it's not clear to me that for webui many of them matter, but probably
I'm missing some.

+    head.headers = headers;
+    head.mime_type = source->source()->GetMimeType(path);
+    // TODO: fill all the time related field i.e. request_time response_time
+    // request_start response_start
+    client_->OnReceiveResponse(head, base::nullopt,
+                               mojom::DownloadedTempFilePtr());

[yzshen1, 2017/05/05 19:54:07]

nit, fyi: you could use nullptr instead of mojom::DownloadedTempFilePtr().

[jam, 2017/05/05 22:19:39]

Done.

+
+    ResourceRequestInfo::WebContentsGetter wc_getter =
+        base::Bind(&GetWebContentsFromFTNID, frame_tree_node_id);
+
+    // Forward along the request to the data source.
+    // TODO(jam): once we only have this code path for WebUI, and not the
+    // URLLRequestJob one, then we should switch data sources to run on the UI
+    // thread by default.
+    scoped_refptr<base::SingleThreadTaskRunner> target_runner =
+        source->source()->TaskRunnerForRequestPath(path);
+    if (!target_runner) {
+      source->source()->StartDataRequest(
+          path, wc_getter,
+          base::Bind(&URLLoaderImpl::DataAvailable,
+                     weak_factory_.GetWeakPtr()));
+    } else {
+      // The DataSource wants StartDataRequest to be called on a specific
+      // thread, usually the UI thread, for this path.
+      target_runner->PostTask(
+          FROM_HERE,
+          base::Bind(&URLDataSource::StartDataRequest,
+                     base::Unretained(source->source()), path, wc_getter,

[scottmg, 2017/05/05 19:27:42]

This was doing a RetainedRef on source before (I think?), is the Unretained()
safe here?

[jam, 2017/05/05 22:19:40]

Thanks, I missed that in the other code path. Fixed.

+                     base::Bind(&URLLoaderImpl::DataAvailableOnTargetThread,
+                                weak_factory_.GetWeakPtr())));
+    }
+  }
+
+  void FollowRedirect() override { NOTREACHED(); }
+  void SetPriority(net::RequestPriority priority,
+                   int32_t intra_priority_value) override {
+    NOTREACHED();
+  }
+
+  static void DataAvailableOnTargetThread(
+      const base::WeakPtr<URLLoaderImpl>& weak_ptr,
+      scoped_refptr<base::RefCountedMemory> bytes) {
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::Bind(&URLLoaderImpl::DataAvailable, weak_ptr, bytes));
+  }
+
+  uint32_t GetUnGzippedSize(

[scottmg, 2017/05/05 19:27:42]

Could you use third_party/zlib/google/compression_utils instead of doing these
inline here?

[jam, 2017/05/05 22:19:40]

Ah, didn't know about that (this came from the old code that used to ungzip in
webui).

I'll tweak the third_party versions a bit to avoid extra copies.

+      const scoped_refptr<base::RefCountedMemory>& bytes) {
+    CHECK(bytes->size() >= 4);
+    // Size of output comes from footer of gzip file format, found as the last 4
+    // bytes in the compressed file, which are stored little endian.
+    return base::ByteSwapToLE32(
+        *reinterpret_cast<const uint32_t*>(&bytes->front()[bytes->size() - 4]));
+  }
+
+  void UnGzip(const scoped_refptr<base::RefCountedMemory>& bytes,
+              void* output,
+              uint32_t output_size) {
+    z_stream inflateStream;

[scottmg, 2017/05/05 19:27:42]

inflateStream -> inflate_stream

[jam, 2017/05/05 22:19:40]

(removed this code per your other comment)

+    memset(&inflateStream, 0, sizeof(inflateStream));
+    inflateStream.avail_in = bytes->size();
+    inflateStream.next_in = const_cast<Bytef*>(bytes->front());
+    inflateStream.avail_out = output_size;
+    inflateStream.next_out = reinterpret_cast<Bytef*>(output);
+
+    CHECK(inflateInit2(&inflateStream, 16) == Z_OK);

[scottmg, 2017/05/05 19:27:43]

Where'd the 16 come from?
https://cs.chromium.org/chromium/src/net/filter/gzip_source_stream.cc?rcl=9a0...
uses -15 I think.

+    CHECK(inflate(&inflateStream, Z_FINISH) == Z_STREAM_END);
+    CHECK(inflateEnd(&inflateStream) == Z_OK);
+  }
+
+  void DataAvailable(scoped_refptr<base::RefCountedMemory> bytes) {

[mmenke, 2017/05/05 19:34:15]

Is this a memory mapped file?  I seem to recall it is, so we shouldn't be
copying from it on a named thread.

[jam, 2017/05/05 22:19:40]

This is what the other code does on the IO thread as well.

[mmenke, 2017/05/05 22:59:27]

I don't think it does.  See  URLRequestChromeJob::PostReadTask / CopyData (Which
is run on a separate thread).

[jam, 2017/05/05 23:25:07]

ah, thanks I misread that code then. I'll take care of this in a followup.

+    if (!bytes) {
+      OnError(net::ERR_FAILED);
+      return;
+    }
+
+    if (replacements_) {

[scottmg, 2017/05/05 19:27:42]

Is it going to do this on all response types? Should it be limited to text/html
or something in case whatever our template expansion characters are appear in a
.png?

[jam, 2017/05/05 22:19:39]

Good point, the other code did this too. I didn't think code handling non-html
types would use this, but good to guard the same way.

+      std::string temp_string;
+      // We won't know the the final output size ahead of time, so we have to
+      // use an intermediate string.
+      base::StringPiece source;
+      std::string temp_str;
+      if (gzipped_) {
+        temp_str.resize(GetUnGzippedSize(bytes));
+        UnGzip(bytes, &temp_str[0], temp_str.size());
+        source.set(temp_str.c_str(), temp_str.size());
+        gzipped_ = false;
+      } else {
+        source.set(reinterpret_cast<const char*>(bytes->front()),
+                   bytes->size());
+      }
+      temp_str = ui::ReplaceTemplateExpressions(source, *replacements_);
+      bytes = base::RefCountedString::TakeString(&temp_str);
+    }
+
+    uint32_t output_size = gzipped_ ? GetUnGzippedSize(bytes) : bytes->size();
+
+    MojoCreateDataPipeOptions options;
+    options.struct_size = sizeof(MojoCreateDataPipeOptions);
+    options.flags = MOJO_CREATE_DATA_PIPE_OPTIONS_FLAG_NONE;
+    options.element_num_bytes = 1;
+    options.capacity_num_bytes = output_size;
+    mojo::DataPipe data_pipe(options);
+
+    DCHECK(data_pipe.producer_handle.is_valid());
+    DCHECK(data_pipe.consumer_handle.is_valid());
+
+    void* buffer = nullptr;
+    uint32_t num_bytes = output_size;
+    MojoResult result =
+        BeginWriteDataRaw(data_pipe.producer_handle.get(), &buffer, &num_bytes,

[mmenke, 2017/05/05 19:34:15]

Is this guaranteed to work?  Is output_sized guaranteed to be something
reasonable to fit into browser memory at once?

[jam, 2017/05/05 22:19:39]

Good question, I wondered about that as well, but then I realized:
-the other code used to decompress at once
-the sizes in https://bugs.chromium.org/p/chromium/issues/detail?id=609219 show
the bigggest as 1.27M (granted, from a year ago), with the only a few at a few
houndred KB and the rest XX KB.
-the default data pipe/resourcebuffer size is half a meg

So I figured we're safe for now, and if if it's wrong then the CHECK will alert
us.

+                          MOJO_WRITE_DATA_FLAG_NONE);
+    CHECK_EQ(result, MOJO_RESULT_OK);
+    CHECK_EQ(num_bytes, output_size);
+
+    if (gzipped_) {
+      UnGzip(bytes, buffer, output_size);
+    } else {
+      memcpy(buffer, bytes->front(), output_size);
+    }
+    result = EndWriteDataRaw(data_pipe.producer_handle.get(), num_bytes);
+    CHECK_EQ(result, MOJO_RESULT_OK);
+
+    client_->OnStartLoadingResponseBody(std::move(data_pipe.consumer_handle));
+
+    ResourceRequestCompletionStatus request_complete_data;
+    request_complete_data.error_code = net::OK;
+    request_complete_data.exists_in_cache = false;
+    request_complete_data.completion_time = base::TimeTicks::Now();
+    request_complete_data.encoded_data_length = output_size;
+    request_complete_data.encoded_body_length = output_size;
+    client_->OnComplete(request_complete_data);
+    delete this;
+  }
+
+  void OnError(int error_code) {
+    ResourceRequestCompletionStatus status;
+    status.error_code = error_code;
+    client_->OnComplete(status);
+    delete this;
+  }
+
+  mojo::AssociatedBinding<mojom::URLLoader> binding_;
+  mojom::URLLoaderClientPtr client_;
+  bool gzipped_;
+  // Replacement dictionary for i18n.
+  const ui::TemplateReplacements* replacements_;
+  base::WeakPtrFactory<URLLoaderImpl> weak_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(URLLoaderImpl);
+};
+
+class WebUIURLLoaderFactory : public mojom::URLLoaderFactory,
+                              public FrameTreeNode::Observer {
+ public:
+  WebUIURLLoaderFactory(FrameTreeNode* ftn)
+      : frame_tree_node_id_(ftn->frame_tree_node_id()),
+        resource_context_(ftn->current_frame_host()
+                              ->GetProcess()
+                              ->GetBrowserContext()
+                              ->GetResourceContext()) {
+    ftn->AddObserver(this);
+  }
+
+  ~WebUIURLLoaderFactory() override {}
+
+  mojom::URLLoaderFactoryPtr CreateBinding() {
+    return loader_factory_bindings_.CreateInterfacePtrAndBind(this);
+  }
+
+  // mojom::URLLoaderFactory implementation:
+  void CreateLoaderAndStart(mojom::URLLoaderAssociatedRequest loader,
+                            int32_t routing_id,
+                            int32_t request_id,
+                            uint32_t options,
+                            const ResourceRequest& request,
+                            mojom::URLLoaderClientPtr client) override {
+    DCHECK_CURRENTLY_ON(BrowserThread::UI);
+    BrowserThread::PostTask(
+        BrowserThread::IO, FROM_HERE,
+        base::BindOnce(&URLLoaderImpl::Create, std::move(loader), request,
+                       frame_tree_node_id_, client.PassInterface(),
+                       resource_context_));
+  }
+
+  void SyncLoad(int32_t routing_id,
+                int32_t request_id,
+                const ResourceRequest& request,
+                SyncLoadCallback callback) override {
+    NOTREACHED();
+  }
+
+  // FrameTreeNode::Observer implementation:
+  void OnFrameTreeNodeDestroyed(FrameTreeNode* node) override {
+    g_factories.Get().erase(frame_tree_node_id_);
+  }
+
+ private:
+  int frame_tree_node_id_;
+  ResourceContext* resource_context_;
+  mojo::BindingSet<mojom::URLLoaderFactory> loader_factory_bindings_;
+
+  DISALLOW_COPY_AND_ASSIGN(WebUIURLLoaderFactory);
+};
+
+}  // namespace
+
+mojom::URLLoaderFactoryPtr GetWebUIURLLoader(FrameTreeNode* node) {
+  int ftn_id = node->frame_tree_node_id();
+  if (g_factories.Get()[ftn_id].get() == nullptr)
+    g_factories.Get()[ftn_id] = base::MakeUnique<WebUIURLLoaderFactory>(node);
+  return g_factories.Get()[ftn_id]->CreateBinding();
+}
+
+}  // namespace content