Implement exponential back-off mechanism and enforce it at the URLRequestHttpJob level.

net/request_throttler/request_throttler_manager.cc

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/request_throttler/request_throttler_manager.h"
+
+#include "base/string_util.h"
+
+const unsigned int RequestThrottlerManager::kMaximumNumberOfEntries = 1500;
+const unsigned int RequestThrottlerManager::kRequestsBetweenCollecting = 200;
+
+scoped_refptr<RequestThrottlerEntryInterface>
+    RequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+
+  AutoLock auto_lock(lock_);
+
+  // Periodically garbage collect old entries.
+  GarbageCollectEntriesIfNecessary();
+
+  // Find the entry in the map or create it.
+  scoped_refptr<RequestThrottlerEntry>& entry = url_entries_[url_id];
+  if (entry == NULL) {
+    if (!no_throttling_by_default_) {
+      entry = new RequestThrottlerEntry();
+    } else {
+      // In order to disable back-off, create a very permissive instance.
+      // This is used by unit tests since we would like to avoid indeterministic
+      // behavior in unit tests.
+      entry = new RequestThrottlerEntry(1, 10000, 0, 0, 1.0, 0.0, 0, 120000);
+    }
+  }
+
+  return entry;
+}
+
+RequestThrottlerManager::RequestThrottlerManager()
+    : requests_since_last_gc_(0),
+      no_throttling_by_default_(false) {
+}
+
+RequestThrottlerManager::~RequestThrottlerManager() {
+  // Delete all entries.
+  url_entries_.clear();
+}
+
+std::string RequestThrottlerManager::GetIdFromUrl(const GURL& url) {
+  std::string url_id;
+  url_id += url.scheme();

[eroman, 2010/11/11 17:54:21]

Is this going to be reached for data: URLs? We wouldn't ever want those to get
throttled. Same thing for externally registered schemes.

[yzshen, 2010/11/11 22:17:05]

At URLRequest(Job) level, only URLRequestHttpJob enforces exponential back-off
and rejects outgoing requests when server errors are encountered. Other schemes
won't be affected.

The only scenario I can think up in which RequestThrottlerEntry/Manager can
affect other schemes: use URLFetcher to fetch URLs with schemes other than
http/https. In that case, the sliding window will apply to those requests.
However, I don't think that it is a problem: it behaves in the same way as
before.

+  url_id += "://";
+  url_id += url.host();
+  url_id += url.path();
+
+  return StringToLowerASCII(url_id);
+}
+
+void RequestThrottlerManager::GarbageCollectEntries() {
+  lock_.AssertAcquired();
+
+  UrlEntryMap::iterator i = url_entries_.begin();
+
+  while (i != url_entries_.end()) {
+    if ((i->second)->IsEntryOutdated()) {
+      url_entries_.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+
+  // In case something broke we want to make sure not to grow indefinitely.
+  while (url_entries_.size() > kMaximumNumberOfEntries) {
+    url_entries_.erase(url_entries_.begin());
+  }
+}
+
+void RequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
+  lock_.AssertAcquired();
+
+  requests_since_last_gc_++;
+  if (requests_since_last_gc_ < kRequestsBetweenCollecting)
+    return;
+
+  requests_since_last_gc_ = 0;
+  GarbageCollectEntries();
+}
+
+void RequestThrottlerManager::NotifyRequestBodyWasMalformed(const GURL& url) {
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+
+  AutoLock auto_lock(lock_);
+
+  UrlEntryMap::iterator i = url_entries_.find(url_id);
+  if (i != url_entries_.end()) {
+    i->second->ReceivedContentWasMalformed();
+  }
+}
+
+int64 RequestThrottlerManager::GetBackoffDelayInMilliseconds(const GURL& url) {
+  std::string url_id = GetIdFromUrl(url);
+  base::TimeTicks release_time;
+  base::TimeTicks now = base::TimeTicks::Now();
+
+  {
+    AutoLock auto_lock(lock_);
+
+    UrlEntryMap::iterator i = url_entries_.find(url_id);
+    release_time = i != url_entries_.end() ? i->second->release_time() : now;
+  }
+
+  base::TimeDelta delay = std::max(release_time - now, base::TimeDelta());
+  return delay.InMillisecondsRoundedUp();
+}
+
+void RequestThrottlerManager::OverrideEntry(
+    const GURL& url,
+    const scoped_refptr<RequestThrottlerEntry>& entry) {
+  if (entry == NULL)
+    return;
+
+  // Normalize the url.
+  std::string url_id = GetIdFromUrl(url);
+
+  AutoLock auto_lock(lock_);
+
+  // Periodically garbage collect old entries.
+  GarbageCollectEntriesIfNecessary();
+
+  url_entries_[url_id] = entry;
+}
+
+void RequestThrottlerManager::ClearAllEntries() {
+  AutoLock auto_lock(lock_);
+  url_entries_.clear();
+}
+
+void RequestThrottlerManager::DisableThrottlingByDefault() {
+  AutoLock auto_lock(lock_);
+  no_throttling_by_default_ = true;
+}