Implement exponential back-off mechanism and enforce it at the URLRequestHttpJob level.

net/url_request/request_throttler_manager.cc

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/url_request/request_throttler_manager.h"
+
+#include "base/string_util.h"
+
+const unsigned int RequestThrottlerManager::kMaximumNumberOfEntries = 1500;
+const unsigned int RequestThrottlerManager::kRequestsBetweenCollecting = 200;
+
+scoped_refptr<RequestThrottlerEntryInterface>
+    RequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+
+  AutoLock auto_lock(lock_);

[willchan no longer on Chromium, 2010/11/18 06:47:47]

Why do we need all this locking?  Shouldn't the IO thread be the only thing that
accesses the RequestThrottleManager?

[yzshen, 2010/11/19 23:51:36]

I have eliminated all locks. Thanks!

+
+  // Periodically garbage collect old entries.
+  GarbageCollectEntriesIfNecessary();
+
+  // Find the entry in the map or create it.
+  scoped_refptr<RequestThrottlerEntry>& entry = url_entries_[url_id];
+  if (entry == NULL)
+    entry = new RequestThrottlerEntry();
+
+  return entry;
+}
+
+RequestThrottlerManager::RequestThrottlerManager()
+    : requests_since_last_gc_(0) {
+}
+
+RequestThrottlerManager::~RequestThrottlerManager() {
+  // Delete all entries.
+  url_entries_.clear();
+}
+
+std::string RequestThrottlerManager::GetIdFromUrl(const GURL& url) {
+  std::string url_id;
+  url_id += url.scheme();
+  url_id += "://";
+  url_id += url.host();
+  url_id += url.path();
+
+  return StringToLowerASCII(url_id);
+}
+
+void RequestThrottlerManager::GarbageCollectEntries() {
+  lock_.AssertAcquired();
+
+  UrlEntryMap::iterator i = url_entries_.begin();
+
+  while (i != url_entries_.end()) {
+    if ((i->second)->IsEntryOutdated()) {
+      url_entries_.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+
+  // In case something broke we want to make sure not to grow indefinitely.
+  while (url_entries_.size() > kMaximumNumberOfEntries) {
+    url_entries_.erase(url_entries_.begin());
+  }
+}
+
+void RequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
+  lock_.AssertAcquired();
+
+  requests_since_last_gc_++;
+  if (requests_since_last_gc_ < kRequestsBetweenCollecting)
+    return;
+
+  requests_since_last_gc_ = 0;
+  GarbageCollectEntries();
+}
+
+void RequestThrottlerManager::OverrideEntry(
+    const GURL& url,
+    const scoped_refptr<RequestThrottlerEntry>& entry) {
+  if (entry == NULL)
+    return;
+
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+
+  AutoLock auto_lock(lock_);
+
+  // Periodically garbage collect old entries.
+  GarbageCollectEntriesIfNecessary();
+
+  url_entries_[url_id] = entry;
+}
+
+void RequestThrottlerManager::EraseEntry(const GURL& url) {
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+
+  AutoLock auto_lock(lock_);
+
+  url_entries_.erase(url_id);
+}