DuplicateContentResourceHandler to monitor resources and track how many times th…

content/browser/renderer_host/duplicate_resource_handler.cc

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/renderer_host/duplicate_resource_handler.h"
+
+#include <set>
+
+#include "base/logging.h"
+#include "base/memory/singleton.h"
+#include "base/metrics/histogram.h"
+#include "base/time.h"
+#include "content/browser/renderer_host/resource_request_info_impl.h"
+#include "net/base/io_buffer.h"
+#include "net/url_request/url_request.h"
+#include "third_party/smhasher/src/PMurHash.h"
+
+using base::TimeTicks;
+
+namespace content {
+
+namespace {
+
+class GlobalDuplicateRecords {
+  public:
+    static GlobalDuplicateRecords* GetInstance() {
+      return Singleton<GlobalDuplicateRecords>::get();
+    }
+
+    std::set<uint32>* content_matches() {
+      return &content_matches_;
+    }
+
+    std::set<uint32>* content_and_url_matches() {
+      return &content_and_url_matches_;
+    }
+
+    int64 browser_start_time() {
+      return browser_start_time_;
+    }
+
+  private:
+    friend class Singleton<GlobalDuplicateRecords>;
+    friend struct DefaultSingletonTraits<GlobalDuplicateRecords>;
+
+    GlobalDuplicateRecords()
+        : browser_start_time_(TimeTicks::Now().ToInternalValue()) {
+    }
+    ~GlobalDuplicateRecords() {}
+
+    std::set<uint32> content_matches_;
+    std::set<uint32> content_and_url_matches_;
+    int64 browser_start_time_;

[gavinp, 2012/07/20 17:58:17]

Why isn't this of type base::TimeTicks ?

+};
+
+}  // namespace
+
+DuplicateResourceHandler::DuplicateResourceHandler(
+    scoped_ptr<ResourceHandler> next_handler,
+    ResourceType::Type resource_type,
+    net::URLRequest* request)
+    : LayeredResourceHandler(next_handler.Pass()),
+      resource_type_(resource_type),
+      bytes_read_(0),
+      request_(request),
+      pmurhash_ph1_(0),
+      pmurhash_pcarry_(0) {
+}
+
+DuplicateResourceHandler::~DuplicateResourceHandler() {
+}
+
+bool DuplicateResourceHandler::OnWillRead(int request_id, net::IOBuffer** buf,
+                                          int* buf_size, int min_size) {
+  DCHECK_EQ(-1, min_size);
+
+  if (!next_handler_->OnWillRead(request_id, buf, buf_size, min_size))
+    return false;
+  read_buffer_ = *buf;
+  return true;
+}
+
+bool DuplicateResourceHandler::OnReadCompleted(int request_id, int bytes_read,
+                                              bool* defer) {
+  PMurHash32_Process(&pmurhash_ph1_, &pmurhash_pcarry_,
+                      read_buffer_->data(), bytes_read);
+  bytes_read_ += bytes_read;
+  return next_handler_->OnReadCompleted(request_id, bytes_read, defer);
+}
+
+bool DuplicateResourceHandler::OnResponseCompleted(
+    int request_id,
+    const net::URLRequestStatus& status,
+    const std::string& security_info) {
+
+  if (status.status() != net::URLRequestStatus::SUCCESS)
+    return next_handler_->OnResponseCompleted(request_id,
+                                              status, security_info);
+
+  uint32 contents_hash = PMurHash32_Result(pmurhash_ph1_,
+                                           pmurhash_pcarry_, bytes_read_);
+
+  // Combine the contents_hash with the url, so we can test if future content
+  // identical resources have the same original url or not.
+  uint32 hashed_with_url;
+  const std::string url_spec = request_->url().spec();
+  PMurHash32_Process(&pmurhash_ph1_, &pmurhash_pcarry_,
+                     url_spec.data(), url_spec.length());
+  hashed_with_url = PMurHash32_Result(pmurhash_ph1_, pmurhash_pcarry_,
+                                      url_spec.length() + bytes_read_);
+
+  DVLOG(4) << "url: " << url_spec;
+  DVLOG(4) << "contents hash: " << contents_hash;
+  DVLOG(4) << "hash with url: " << hashed_with_url;
+
+  std::set<uint32>* content_matches =
+      GlobalDuplicateRecords::GetInstance()->content_matches();
+  std::set<uint32>* content_and_url_matches =
+      GlobalDuplicateRecords::GetInstance()->content_and_url_matches();
+
+  const bool did_match_contents = content_matches->count(contents_hash);
+  const bool did_match_contents_and_url =
+      content_and_url_matches->count(hashed_with_url);
+
+  UMA_HISTOGRAM_BOOLEAN("Duplicate.Hits", did_match_contents);
+  UMA_HISTOGRAM_BOOLEAN("Duplicate.HitsSameUrl", did_match_contents &&
+                         did_match_contents_and_url);
+  if (did_match_contents && !did_match_contents_and_url) {
+      int64 session_length = TimeTicks::Now().ToInternalValue() -
+          GlobalDuplicateRecords::GetInstance()->browser_start_time();

[gavinp, 2012/07/20 17:58:17]

Why isn't this of type base::TimeDelta ?

+      content_and_url_matches->insert(hashed_with_url);
+      UMA_HISTOGRAM_CUSTOM_COUNTS("Duplicate.Size.HashHitUrlMiss", bytes_read_,
+                                   1, 0x7FFFFFFF, 50);
+      UMA_HISTOGRAM_ENUMERATION("Duplicate.ResourceType.HashHitUrlMiss",
+                                 resource_type_, ResourceType::LAST_TYPE);
+      UMA_HISTOGRAM_CUSTOM_COUNTS("Duplicate.SessionLength.HashHitUrlMiss",
+                                   session_length/(1000*1000),
+                                   1, 0x7FFFFFFF, 50);

[gavinp, 2012/07/20 17:58:17]

Isn't this only useful if you have Duplicate.BySessionLength.<someotherthing> to
compare it to? Can you explain the use of this without one of the other ones to
compare to?

[gavinp, 2012/07/20 23:30:51]

I'm sad to see this idea gone. I think it's most interesting not with respect to
time, but with respect to the number of resources retrieved. In particular, I'd
like to gather enough data to estimate the equilibrium hit rate on each of these
variables. With the data we have here, it's unclear how we'll separate newly
started browsers (with many misses) from the older browsers with equilibrium
rates.

+  }
+  content_matches->insert(contents_hash);
+  content_and_url_matches->insert(hashed_with_url);
+
+  bytes_read_ = 0;
+  read_buffer_ = NULL;
+  return next_handler_->OnResponseCompleted(request_id, status, security_info);
+}
+
+}  // namespace content
+