Implement exponential back-off mechanism and enforce it at the URLRequestHttpJob level.

net/url_request/request_throttler_manager.cc

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/url_request/request_throttler_manager.h"
+
+#include "base/string_util.h"
+
+const unsigned int RequestThrottlerManager::kMaximumNumberOfEntries = 1500;
+const unsigned int RequestThrottlerManager::kRequestsBetweenCollecting = 200;
+
+RequestThrottlerManager* RequestThrottlerManager::GetInstance() {
+  return Singleton<RequestThrottlerManager>::get();
+}
+
+scoped_refptr<RequestThrottlerEntryInterface>
+    RequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+
+  // Periodically garbage collect old entries.
+  GarbageCollectEntriesIfNecessary();
+
+  // Find the entry in the map or create it.
+  scoped_refptr<RequestThrottlerEntry>& entry = url_entries_[url_id];
+  if (entry == NULL)
+    entry = new RequestThrottlerEntry();
+
+  return entry;
+}
+
+RequestThrottlerManager::RequestThrottlerManager()
+    : requests_since_last_gc_(0) {
+}
+
+RequestThrottlerManager::~RequestThrottlerManager() {
+  // Delete all entries.
+  url_entries_.clear();
+}
+
+std::string RequestThrottlerManager::GetIdFromUrl(const GURL& url) {

[willchan no longer on Chromium, 2010/11/24 00:23:07]

Why are you not using the GURL canonicalization routines?  What if the URL has a
port or query or username or password or what not?  The port especially seems
important.  It seems to me that you should use url_canon::Replacements to
generate a new GURL and use .spec() to convert.

[yzshen, 2010/11/24 09:42:08]

I have changed to use Replacements to convert. Thanks!

I agree with you that port is important, I have added it. 
For other parts, it is intended to throw them away.

+  std::string url_id;
+  url_id += url.scheme();
+  url_id += "://";
+  url_id += url.host();
+  url_id += url.path();
+
+  return StringToLowerASCII(url_id);
+}
+
+void RequestThrottlerManager::GarbageCollectEntries() {
+  UrlEntryMap::iterator i = url_entries_.begin();
+
+  while (i != url_entries_.end()) {
+    if ((i->second)->IsEntryOutdated()) {
+      url_entries_.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+
+  // In case something broke we want to make sure not to grow indefinitely.
+  while (url_entries_.size() > kMaximumNumberOfEntries) {
+    url_entries_.erase(url_entries_.begin());
+  }
+}
+
+void RequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
+  requests_since_last_gc_++;
+  if (requests_since_last_gc_ < kRequestsBetweenCollecting)
+    return;
+
+  requests_since_last_gc_ = 0;
+  GarbageCollectEntries();
+}
+
+void RequestThrottlerManager::OverrideEntryForTests(
+    const GURL& url,
+    RequestThrottlerEntry* entry) {
+  if (entry == NULL)
+    return;
+
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+
+  // Periodically garbage collect old entries.
+  GarbageCollectEntriesIfNecessary();
+
+  url_entries_[url_id] = entry;
+}
+
+void RequestThrottlerManager::EraseEntryForTests(const GURL& url) {
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+  url_entries_.erase(url_id);
+}