Adds a --deterministic-fetch flag to headless_shell

headless/public/util/http_url_fetcher.cc

Index: headless/public/util/http_url_fetcher.cc
diff --git a/headless/public/util/http_url_fetcher.cc b/headless/public/util/http_url_fetcher.cc
new file mode 100644
index 0000000000000000000000000000000000000000..311e9783a65198241955bb5e258638fed06519dc
--- /dev/null
+++ b/headless/public/util/http_url_fetcher.cc
@@ -0,0 +1,192 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "headless/public/util/http_url_fetcher.h"
+
+#include "net/base/io_buffer.h"
+#include "net/cert/cert_status_flags.h"
+#include "net/url_request/url_request.h"
+#include "net/url_request/url_request_context.h"
+#include "net/url_request/url_request_job_factory_impl.h"
+
+namespace headless {
+
+class HttpUrlFetcher::Delegate : public net::URLRequest::Delegate {
+ public:
+  Delegate(const GURL& rewritten_url,
+           const net::HttpRequestHeaders& request_headers,
+           net::URLRequestContext* url_request_context,
+           ResultListener* result_listener);
+  ~Delegate() override;
+
+  // URLRequest::Delegate methods:
+  void OnAuthRequired(net::URLRequest* request,
+                      net::AuthChallengeInfo* auth_info) override;
+  void OnSSLCertificateError(net::URLRequest* request,
+                             const net::SSLInfo& ssl_info,
+                             bool is_hsts_ok) override;
+  void OnResponseStarted(net::URLRequest* request, int net_error) override;
+  void OnReadCompleted(net::URLRequest* request, int num_bytes) override;
+
+ private:
+  enum { kBufSize = 4096 };
+
+  bool ConsumeBytesRead(net::URLRequest* request, int num_bytes);
+  void ReadBody(net::URLRequest* request);
+  void OnResponseCompleted(net::URLRequest* request, int net_error);
+
+  // Holds the error condition that was hit by the request, or OK.
+  int result_code_;
+
+  // Buffer that URLRequest writes into.
+  scoped_refptr<net::IOBuffer> buf_;
+
+  // The HTTP fetch.
+  std::unique_ptr<net::URLRequest> request_;
+
+  // The Job factory.
+  std::unique_ptr<net::URLRequestJobFactory> job_factory_;
+
+  // Holds the bytes read so far.
+  std::string bytes_read_so_far_;
+
+  // The interface kn which to report any results.
+  ResultListener* result_listener_;  // NOT OWNED.
+};
+
+HttpUrlFetcher::Delegate::Delegate(
+    const GURL& rewritten_url,
+    const net::HttpRequestHeaders& request_headers,
+    net::URLRequestContext* url_request_context,
+    ResultListener* result_listener)
+    : result_code_(net::OK),
+      buf_(new net::IOBuffer(kBufSize)),
+      request_(url_request_context->CreateRequest(rewritten_url,
+                                                  net::DEFAULT_PRIORITY,
+                                                  this)),
+      result_listener_(result_listener) {
+  // TODO(alexclarke): Support the other HTTP verbs.
+  request_->set_method("GET");

[Sami, 2016/09/22 12:09:40]

Could we add a DCHECK for this somewhere? I think silently turning POSTs into
GETs will be surprising to say the least.

[alex clarke (OOO till 29th), 2016/09/23 13:29:22]

I went ahead and added support for the verb.

+  request_->SetExtraRequestHeaders(request_headers);
+  request_->Start();
+}
+
+HttpUrlFetcher::Delegate::~Delegate() {}
+
+void HttpUrlFetcher::Delegate::OnAuthRequired(
+    net::URLRequest* request,
+    net::AuthChallengeInfo* auth_info) {
+  DCHECK_EQ(request, request_.get());
+  LOG(WARNING) << "Auth required to fetch URL, aborting.";
+  result_code_ = net::ERR_NOT_IMPLEMENTED;
+  request->CancelAuth();
+}
+
+void HttpUrlFetcher::Delegate::OnSSLCertificateError(
+    net::URLRequest* request,
+    const net::SSLInfo& ssl_info,
+    bool is_hsts_ok) {
+  DCHECK_EQ(request, request_.get());
+
+  // Revocation check failures are not fatal.
+  if (net::IsCertStatusMinorError(ssl_info.cert_status)) {
+    request->ContinueDespiteLastError();
+    return;
+  }
+  LOG(WARNING) << "SSL certificate error when fetching PAC script, aborting.";

[Sami, 2016/09/22 12:09:40]

I'm not sure how this is related to PAC scripts?

[alex clarke (OOO till 29th), 2016/09/23 13:29:22]

Whoops cut & paste fail.

+
+  // Certificate errors are in same space as net errors.
+  result_code_ = net::MapCertStatusToNetError(ssl_info.cert_status);
+  request->Cancel();
+}
+
+void HttpUrlFetcher::Delegate::OnResponseStarted(net::URLRequest* request,
+                                                 int net_error) {
+  DCHECK_EQ(request, request_.get());
+  DCHECK_NE(net::ERR_IO_PENDING, net_error);
+
+  if (net_error != net::OK) {
+    OnResponseCompleted(request, net_error);
+    return;
+  }
+
+  ReadBody(request);
+}
+
+void HttpUrlFetcher::Delegate::OnReadCompleted(net::URLRequest* request,
+                                               int num_bytes) {
+  DCHECK_EQ(request, request_.get());
+  DCHECK_NE(net::ERR_IO_PENDING, num_bytes);
+
+  if (ConsumeBytesRead(request, num_bytes)) {
+    // Keep reading.
+    ReadBody(request);
+  }
+}
+
+void HttpUrlFetcher::Delegate::ReadBody(net::URLRequest* request) {
+  // Read as many bytes as are available synchronously.
+  while (true) {
+    int num_bytes = request->Read(buf_.get(), kBufSize);
+    if (num_bytes == net::ERR_IO_PENDING)
+      return;
+
+    if (num_bytes < 0) {
+      OnResponseCompleted(request, num_bytes);
+      return;
+    }
+
+    if (!ConsumeBytesRead(request, num_bytes))
+      return;
+  }
+}
+
+bool HttpUrlFetcher::Delegate::ConsumeBytesRead(net::URLRequest* request,
+                                                int num_bytes) {
+  if (num_bytes <= 0) {
+    // Error while reading, or EOF.
+    OnResponseCompleted(request, num_bytes);
+    return false;
+  }
+
+  bytes_read_so_far_.append(buf_->data(), num_bytes);
+  return true;
+}
+
+void HttpUrlFetcher::Delegate::OnResponseCompleted(net::URLRequest* request,
+                                                   int net_error) {
+  DCHECK_EQ(request, request_.get());
+
+  if (result_code_ != net::OK) {
+    result_listener_->OnFetchStartError(static_cast<net::Error>(result_code_));
+    return;
+  }
+
+  if (net_error != net::OK) {
+    result_listener_->OnFetchStartError(static_cast<net::Error>(net_error));
+    return;
+  }
+
+  result_listener_->OnFetchCompleteExtractHeaders(
+      request->url(), request->GetResponseCode(), bytes_read_so_far_.c_str(),
+      bytes_read_so_far_.size());
+}
+
+HttpUrlFetcher::HttpUrlFetcher(
+    std::unique_ptr<net::URLRequestContext> url_request_context)
+    : url_request_context_(std::move(url_request_context)),
+      url_request_job_factory_(new net::URLRequestJobFactoryImpl()) {
+  url_request_context_->set_job_factory(url_request_job_factory_.get());
+}
+
+HttpUrlFetcher::~HttpUrlFetcher() {}
+
+void HttpUrlFetcher::StartFetch(const GURL& rewritten_url,
+                                const net::HttpRequestHeaders& request_headers,
+                                ResultListener* result_listener) {
+  delagate_.reset(new Delegate(rewritten_url, request_headers,
+                               url_request_context_.get(), result_listener));
+}
+
+}  // namespace headless