Adds speculative prefetching of resources.

chrome/browser/predictors/resource_prefetcher.cc

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetcher.h"
+
+#include "base/stl_util.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/base/io_buffer.h"
+
+namespace {
+
+// The size of the buffer used to read the resource.
+static const size_t kResourceBufferSizeBytes = 50000;

[dominich, 2012/07/23 16:04:41]

note: histogram the sizes to ensure this is reasonable.

[Shishir, 2012/08/01 22:35:24]

This is not the total size read, but the buffer size. Histogramming this would
be equivalent of histogramming size of resources.

+
+}  // namespace
+
+namespace predictors {
+
+ResourcePrefetcher::Request::Request(const GURL& i_resource_url)
+    : resource_url(i_resource_url),
+      prefetch_status(PREFETCH_STATUS_NOT_STARTED),
+      usage_status(USAGE_STATUS_NOT_REQUESTED) {
+}
+
+ResourcePrefetcher::Request::Request(const Request& other)
+    : resource_url(other.resource_url),
+      prefetch_status(other.prefetch_status),
+      usage_status(other.usage_status) {
+}
+
+ResourcePrefetcher::ResourcePrefetcher(
+    Delegate* delegate,
+    const ResourcePrefetchPredictorConfig& config,
+    const NavigationID& navigation_id,
+    scoped_ptr<RequestVector> requests)
+        : state_(INITIALIZED),
+          delegate_(delegate),
+          config_(config),
+          navigation_id_(navigation_id),
+          request_vector_(requests.Pass()) {

[dominich, 2012/07/23 16:04:41]

DCHECK(request_vector_.get());

[Shishir, 2012/08/01 22:35:24]

Done.

+  CHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::IO));
+
+  for (int i = 0; i < static_cast<int>(request_vector_->size()); ++i) {

[dominich, 2012/07/23 16:04:41]

std::copy(request_vector_->begin(), request_vector->end(),
           std::back_inserter(request_queue_));

[Shishir, 2012/08/01 22:35:24]

Done.

+    request_queue_.push_back(request_vector_->at(i).get());
+  }
+}
+
+ResourcePrefetcher::~ResourcePrefetcher() {
+  // Delete any pending net::URLRequests.
+  STLDeleteContainerPairFirstPointers(inflight_requests_.begin(),
+                                      inflight_requests_.end());
+}
+
+void ResourcePrefetcher::Start() {
+  CHECK(CalledOnValidThread());
+
+  CHECK_EQ(state_, INITIALIZED);
+  state_ = RUNNING;
+
+  TryToLaunchPrefetchRequests();
+}
+
+void ResourcePrefetcher::Stop() {
+  CHECK(CalledOnValidThread());
+
+  if (state_ == FINISHED)
+    return;
+
+  state_ = STOPPED;
+}
+
+void ResourcePrefetcher::TryToLaunchPrefetchRequests() {
+  CHECK(state_ == RUNNING || state_ == STOPPED);
+
+  // Try to launch new requests if the state is RUNNING.
+  if (state_ == RUNNING) {
+    bool request_available = true;
+    while ((inflight_requests_.size() <
+                config_.max_prefetches_inflight_per_navigation) &&

[dominich, 2012/07/23 16:04:41]

consider histogram for this case

[Shishir, 2012/08/01 22:35:24]

I will send out a separate CL with a lot more histograms added.

+           request_available) {
+      std::list<Request*>::iterator it = request_queue_.begin();

[dominich, 2012/07/23 16:04:41]

can you comment this loop? it's a little tricky and explaining that you're
searching for the first host that has an empty slot would clear it up.

[Shishir, 2012/08/01 22:35:24]

Done.

+      for (; it != request_queue_.end(); ++it) {
+        const std::string& host = (*it)->resource_url.host();
+        if (host_inflight_counts_.find(host) == host_inflight_counts_.end() ||
+            host_inflight_counts_[host] <

[dominich, 2012/07/23 16:04:41]

remove double lookup.

[Shishir, 2012/08/01 22:35:24]

Done.

+                config_.max_prefetches_inflight_per_host_per_navigation)

[dominich, 2012/07/23 16:04:41]

consider histogram for this case.

[Shishir, 2012/08/01 22:35:24]

Will send another CL with a lot of histograms :)

+          break;
+      }
+      request_available = it != request_queue_.end();
+
+      if (request_available) {
+        Request* request = *it;
+        request_queue_.erase(it);
+        SendRequest(request);

[dominich, 2012/07/23 16:04:41]

SendRequest(*it);
request_queue_.erase(it);

[Shishir, 2012/08/01 22:35:24]

Done.

+      }
+    }
+  }
+
+  // If the inflight_requests_ is empty, we cant launch any more. Finish.
+  if (inflight_requests_.empty()) {
+    CHECK(host_inflight_counts_.empty());
+    CHECK(request_queue_.empty() || state_ == STOPPED);
+
+    state_ = FINISHED;
+    delegate_->ResourcePrefetcherFinished(this, request_vector_.Pass());

[dominich, 2012/07/23 16:04:41]

hmm, now i'm confused. This doesn't take ownership of the request_vector_? Then
why is it a scoped_ptr? It could be a raw pointer and ownership can be clearer.

[Shishir, 2012/08/01 22:35:24]

Made the interfaces a little cleaner.

+  }
+}
+
+void ResourcePrefetcher::SendRequest(Request* request) {
+  request->prefetch_status = Request::PREFETCH_STATUS_STARTED;
+
+  net::URLRequest* url_request =
+      new net::URLRequest(request->resource_url,
+                          this,
+                          delegate_->GetURLRequestContext());
+  inflight_requests_[url_request] = request;
+  host_inflight_counts_[url_request->original_url().host()] += 1;
+
+  url_request->set_method("GET");
+  url_request->set_first_party_for_cookies(navigation_id_.main_frame_url);
+  url_request->set_referrer(navigation_id_.main_frame_url.spec());
+  url_request->set_priority(net::LOW);
+  url_request->Start();
+}
+
+void ResourcePrefetcher::FinishRequest(net::URLRequest* request,
+                                       Request::PrefetchStatus status) {
+  CHECK(inflight_requests_.find(request) != inflight_requests_.end());
+
+  const std::string host = request->original_url().host();
+  CHECK_GT(host_inflight_counts_[host], 0);

[dominich, 2012/07/23 16:04:41]

please use the iterator throughout this method.

[Shishir, 2012/08/01 22:35:24]

Done.

+  host_inflight_counts_[host] -= 1;
+  if (host_inflight_counts_[host] == 0)
+    host_inflight_counts_.erase(host);
+
+  inflight_requests_[request]->prefetch_status = status;

[dominich, 2012/07/23 16:04:41]

you set the status and then remove it from the map. How does someone then read
that status? This seems pointless.

[Shishir, 2012/08/01 22:35:24]

I remove it from the inflight map. It is still present in the other maps so the
field is still valid.

+  inflight_requests_.erase(request);
+
+  delete request;

[dominich, 2012/07/23 16:04:41]

here you could use a linked_ptr to get this behaviour on erase, but you don't.
I'm very confused about your use of linked_ptr.

[Shishir, 2012/08/01 22:35:24]

Well no more linked ptrs.

+
+  TryToLaunchPrefetchRequests();
+}
+
+void ResourcePrefetcher::ReadFullResponse(net::URLRequest* request) {
+  bool status = true;
+  while (status) {
+    int bytes_read = 0;
+    net::IOBuffer* buffer = new net::IOBuffer(kResourceBufferSizeBytes);
+    status = request->Read(buffer, kResourceBufferSizeBytes, &bytes_read);

[dominich, 2012/07/23 16:04:41]

I don't know enough about URLRequest - does this take ownership of the buffer,
and will it delete it correctly based on status? If we go through this loop more
than once, does it keep the old buffer around or does the buffer get replaced?

It seems to me, from reading this, that this is going to read the resource in
50k chunks until end/failure but it's throwing away the read bytes. What am I
missing?

[Shishir, 2012/08/01 22:35:24]

The buffer is a refcounted, so it will get destroyed when the request is done
reading it. Fixed the declaration of the buffer.

As for the reading of the data, yes we read the data and throw away the buffer.
If we don't read the entire buffer, the resource is not fully downloaded and
inserted into the cache.

+
+    if (status) {
+      if (request->status().error()) {
+        FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+        return;
+      } else if (bytes_read == 0) {
+        if (request->was_cached())
+          FinishRequest(request, Request::PREFETCH_STATUS_FROM_CACHE);
+        else
+          FinishRequest(request, Request::PREFETCH_STATUS_FROM_NETWORK);
+        return;
+      }
+    }
+  }
+}
+
+void ResourcePrefetcher::OnReceivedRedirect(net::URLRequest* request,
+                                            const GURL& new_url,
+                                            bool* defer_redirect) {
+  FinishRequest(request, Request::PREFETCH_STATUS_CANCELLED);

[dominich, 2012/07/23 16:04:41]

It's worth enumerating these cancelation reasons.

[Shishir, 2012/08/01 22:35:24]

Done.

+}
+
+void ResourcePrefetcher::OnAuthRequired(net::URLRequest* request,
+                                        net::AuthChallengeInfo* auth_info) {
+  FinishRequest(request, Request::PREFETCH_STATUS_CANCELLED);
+}
+
+void ResourcePrefetcher::OnCertificateRequested(
+    net::URLRequest* request,
+    net::SSLCertRequestInfo* cert_request_info) {
+  FinishRequest(request, Request::PREFETCH_STATUS_CANCELLED);
+}
+
+void ResourcePrefetcher::OnSSLCertificateError(net::URLRequest* request,
+                                               const net::SSLInfo& ssl_info,
+                                               bool fatal) {
+  FinishRequest(request, Request::PREFETCH_STATUS_CANCELLED);
+}
+
+void ResourcePrefetcher::OnResponseStarted(net::URLRequest* request) {
+  if (request->status().error()) {
+    FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+    return;
+  }
+
+  ReadFullResponse(request);
+}
+
+void ResourcePrefetcher::OnReadCompleted(net::URLRequest* request,
+                                         int bytes_read) {
+  if (request->status().error()) {
+    FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+    return;
+  }
+
+  ReadFullResponse(request);
+}
+
+}  // namespace predictors