Add an opt-out header for HTTP throttling. Never throttle for localhost.

net/url_request/url_request_throttler_manager.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/url_request/url_request_throttler_manager.h"
 
 #include "base/logging.h"
 #include "base/string_util.h"
+#include "net/base/net_util.h"
 
 namespace net {
 
 const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500;
 const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200;
 
 URLRequestThrottlerManager* URLRequestThrottlerManager::GetInstance() {
   return Singleton<URLRequestThrottlerManager>::get();
 }
 
 scoped_refptr<URLRequestThrottlerEntryInterface>
     URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
   DCHECK(being_tested_ || CalledOnValidThread());
 
   // Normalize the url.
   std::string url_id = GetIdFromUrl(url);
 
   // Periodically garbage collect old entries.
   GarbageCollectEntriesIfNecessary();
 
   // Find the entry in the map or create it.
   scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id];
   if (entry.get() == NULL)
-    entry = new URLRequestThrottlerEntry();
+    entry = new URLRequestThrottlerEntry(this);
+
+  std::string host = url.host();
+  if (opt_out_hosts_.find(host) != opt_out_hosts_.end() || IsLocalhost(host)) {

[yzshen1, 2011/03/18 22:49:51]

[Don't have a strong opinion. Just to make sure you have considered it.]
If we disable throttling for localhost, a web developer, who always tests
his/her website on localhost, may have different experience comparing with his
users. He may be unaware of the throttling feature and doesn't do anything about
it. He may not be able to test opt-out headers. Will that cause any problem?

[Jói, 2011/03/23 23:38:24]

I'm doing this on request of the web developers in crbug.com/66062.  They can
still test exponential back-off locally by using a non-loopback address to point
to their own machine.

+    // TODO(joi): Once sliding window is separate from back-off throttling,
+    // we can simply return a dummy implementation of
+    // URLRequestThrottlerEntryInterface here that never blocks anything (and
+    // not keep entries in url_entries_ for opted-out sites).

[yzshen1, 2011/03/18 22:49:51]

We might still need to keep track of back-off period, because the retry logic in
URLFetcher will need to use the back-off period.

But this is not very important. We may consider to remove the retry logic
entirely as we discussed last time, or simply retry with a fixed interval. The
server won't mind how frequently we retry, because it asks us to disable
throttling at our side.

[Jói, 2011/03/23 23:38:24]

Right, I think as per the comment I added on
URLRequestThrottlerEntry::GetExponentialBackoffReleaseTime, we should not return
the calculated back-off time for hosts that have opted out (instead we should
just return "now").

+    entry->DisableBackoffThrottling();
+  }
 
   return entry;
 }
 
+void URLRequestThrottlerManager::AddToOptOutList(const std::string& host) {
+  // There is an edge case here that we are not handling, to keep things
+  // simple.  If a host starts adding the opt-out header to its responses
+  // after there are already one or more entries in url_entries_ for that
+  // host, the pre-existing entries may still perform back-off throttling.
+  // In practice, this would almost never occur.
+  opt_out_hosts_.insert(host);
+}
+
 void URLRequestThrottlerManager::OverrideEntryForTests(
     const GURL& url,
     URLRequestThrottlerEntry* entry) {
   // Normalize the url.
   std::string url_id = GetIdFromUrl(url);
 
   // Periodically garbage collect old entries.
   GarbageCollectEntriesIfNecessary();
 
   url_entries_[url_id] = entry;
@@ skipping 74 matching lines @@
     }
   }
 
   // In case something broke we want to make sure not to grow indefinitely.
   while (url_entries_.size() > kMaximumNumberOfEntries) {
     url_entries_.erase(url_entries_.begin());
   }
 }
 
 }  // namespace net