Implement exponential back-off mechanism and enforce it at the URLRequestHttpJob level.

net/url_request/request_throttler_manager.h

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_URL_REQUEST_REQUEST_THROTTLER_MANAGER_H_
+#define NET_URL_REQUEST_REQUEST_THROTTLER_MANAGER_H_
+
+#include <map>
+#include <string>
+
+#include "base/lock.h"
+#include "base/scoped_ptr.h"
+#include "base/singleton.h"
+#include "googleurl/src/gurl.h"
+#include "net/url_request/request_throttler_entry.h"
+
+// Class that registers a request throttler entry for each URL being accessed in
+// order to supervise traffic. URL requests for HTTP contents should register
+// their URLs in this request throttler manager on each request.

[willchan no longer on Chromium, 2010/11/18 06:47:47]

This comment is insufficient.  It doesn't describe the garbage collection at
all.

[yzshen, 2010/11/19 23:51:36]

Done.

+class RequestThrottlerManager {
+ public:
+  // Must be called for every request, returns the request throttler entry
+  // associated with the URL. The caller must inform this entry of some events.
+  // Please refer to request_throttler_entry_interface.h for further
+  // informations.
+  scoped_refptr<RequestThrottlerEntryInterface> RegisterRequestUrl(
+      const GURL& url);
+
+  // Registers a new entry in this service and overrides the existing entry (if
+  // any) for the URL.
+  // It is only used by unit tests.

[willchan no longer on Chromium, 2010/11/18 06:47:47]

I think I'd prefer that this be named OverrideEntryForTests() or something if
it's only used by unit tests.  I'd worry about someone not reading the comments
and using this function.

[yzshen, 2010/11/19 23:51:36]

Done.

+  void OverrideEntry(const GURL& url,
+                     const scoped_refptr<RequestThrottlerEntry>& entry);

[willchan no longer on Chromium, 2010/11/17 20:32:34]

This should take a raw pointer rather than a const scoped_refptr<T>&

[yzshen, 2010/11/19 23:51:36]

Done.

+
+  // Explicitly erases an entry.
+  // This is useful to remove those entries which have got infinite lifetime and
+  // thus won't be garbage collected.
+  // It is only used by unit tests.
+  void EraseEntry(const GURL& url);

[willchan no longer on Chromium, 2010/11/18 06:47:47]

Can you name this EraseEntryForTests()?  I read this for the first time in the
code and it wasn't obvious to me it was only for tests.

[yzshen, 2010/11/19 23:51:36]

Done.

+
+ protected:
+  RequestThrottlerManager();
+  virtual ~RequestThrottlerManager();

[willchan no longer on Chromium, 2010/11/17 20:32:34]

Why is the destructor virtual?  I don't see virtual methods.

[yzshen, 2010/11/19 23:51:36]

Done.

+
+  // Method that allows us to transform a URL into an ID that can be used in our
+  // map. Resulting IDs will be lowercase and be missing both the query string
+  // and fragment.
+  std::string GetIdFromUrl(const GURL& url);
+
+  // Method that ensures the map gets cleaned from time to time. The period at
+  // which garbage collecting happens is adjustable with the
+  // kRequestBetweenCollecting constant.
+  void GarbageCollectEntriesIfNecessary();
+  // Method that does the actual work of garbage collecting.
+  void GarbageCollectEntries();
+
+  // From each URL we generate an ID composed of the host and path
+  // that allows us to uniquely map an entry to it.
+  typedef std::map<std::string, scoped_refptr<RequestThrottlerEntry> >

[willchan no longer on Chromium, 2010/11/17 20:32:34]

http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml?showone=Declar...
says types should go at the beginning of the section.

[yzshen, 2010/11/19 23:51:36]

Done.

+      UrlEntryMap;
+
+  friend struct DefaultSingletonTraits<RequestThrottlerManager>;
+
+  // Map that contains a list of URL ID and their matching
+  // RequestThrottlerEntry.
+  UrlEntryMap url_entries_;

[willchan no longer on Chromium, 2010/11/17 20:32:34]

http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml?showone=Class_...
says to make this private.

[yzshen, 2010/11/19 23:51:36]

Done.

+
+  // Lock to protect those non-static data members.
+  Lock lock_;
+
+ private:
+  // Maximum number of entries that we are willing to collect in our map.
+  static const unsigned int kMaximumNumberOfEntries;
+  // Number of requests that will be made between garbage collection.
+  static const unsigned int kRequestsBetweenCollecting;
+
+  // This keeps track of how many requests have been made. Used with
+  // GarbageCollectEntries.
+  unsigned int requests_since_last_gc_;
+
+  DISALLOW_COPY_AND_ASSIGN(RequestThrottlerManager);

[willchan no longer on Chromium, 2010/11/17 20:32:34]

Include basictypes.h for this macro.

[yzshen, 2010/11/19 23:51:36]

Done.

+};
+
+#endif  // NET_URL_REQUEST_REQUEST_THROTTLER_MANAGER_H_