Implement exponential back-off mechanism and enforce it at the URLRequestHttpJob level.

net/url_request/request_throttler_entry.cc

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/url_request/request_throttler_entry.h"
+
+#include <cmath>
+
+#include "base/logging.h"
+#include "base/rand_util.h"
+#include "base/string_number_conversions.h"
+#include "net/url_request/request_throttler_header_interface.h"
+
+const int RequestThrottlerEntry::kDefaultSlidingWindowPeriodMs = 2000;
+const int RequestThrottlerEntry::kDefaultMaxSendThreshold = 20;
+const int RequestThrottlerEntry::kDefaultInitialBackoffMs = 700;
+const int RequestThrottlerEntry::kDefaultAdditionalConstantMs = 100;
+const double RequestThrottlerEntry::kDefaultMultiplyFactor = 2.0;
+const double RequestThrottlerEntry::kDefaultJitterFactor = 0.4;
+const int RequestThrottlerEntry::kDefaultMaximumBackoffMs = 24 * 60 * 60 * 1000;
+const int RequestThrottlerEntry::kDefaultEntryLifetimeMs = 120000;
+const char RequestThrottlerEntry::kRetryHeaderName[] = "X-Retry-After";
+
+RequestThrottlerEntry::RequestThrottlerEntry()
+    : sliding_window_period_ms_(kDefaultSlidingWindowPeriodMs),
+      max_send_threshold_(kDefaultMaxSendThreshold),
+      initial_backoff_ms_(kDefaultInitialBackoffMs),
+      additional_constant_ms_(kDefaultAdditionalConstantMs),
+      multiply_factor_(kDefaultMultiplyFactor),
+      jitter_factor_(kDefaultJitterFactor),
+      maximum_backoff_ms_(kDefaultMaximumBackoffMs),
+      entry_lifetime_ms_(kDefaultEntryLifetimeMs) {
+  Initialize();
+}
+
+RequestThrottlerEntry::RequestThrottlerEntry(
+    int sliding_window_period_ms,
+    int max_send_threshold,
+    int initial_backoff_ms,
+    int additional_constant_ms,
+    double multiply_factor,
+    double jitter_factor,
+    int maximum_backoff_ms)
+    : sliding_window_period_ms_(sliding_window_period_ms),
+      max_send_threshold_(max_send_threshold),
+      initial_backoff_ms_(initial_backoff_ms),
+      additional_constant_ms_(additional_constant_ms),
+      multiply_factor_(multiply_factor),
+      jitter_factor_(jitter_factor),
+      maximum_backoff_ms_(maximum_backoff_ms),
+      entry_lifetime_ms_(-1) {
+  DCHECK(sliding_window_period_ms_ > 0 &&
+         max_send_threshold_ > 0 &&
+         initial_backoff_ms_ >= 0 &&
+         additional_constant_ms_ >= 0 &&
+         multiply_factor_ > 0 &&
+         jitter_factor_ >= 0 &&
+         maximum_backoff_ms_ >= 0);
+
+  Initialize();
+}
+
+RequestThrottlerEntry::~RequestThrottlerEntry() {
+}
+
+void RequestThrottlerEntry::Initialize() {
+  exponential_backoff_release_time_ = base::TimeTicks::Now();

[Jói, 2010/11/12 00:12:08]

suggest using GetTimeNow()

[yzshen, 2010/11/12 02:05:46]

Using GetTimeNow() here is a little bit confusing, since this method is called
in constructors, while in constructors virtual won't work.

I added a comment. Is it okay?

[Jói, 2010/11/12 19:14:24]

That's fine.

+  failure_count_ = 0;
+  tracking_exponential_backoff_ = false;
+
+  old_values_.exponential_backoff_release_time =
+      exponential_backoff_release_time_;
+  old_values_.failure_count = failure_count_;
+
+  sliding_window_release_time_ = base::TimeTicks::Now();

[Jói, 2010/11/12 00:12:08]

likewise

[yzshen, 2010/11/12 02:05:46]

Please see the comment above.

+}
+
+bool RequestThrottlerEntry::IsDuringExponentialBackoff() const {
+  AutoLock auto_lock(lock_);
+  return exponential_backoff_release_time_ > GetTimeNow();
+}
+
+int64 RequestThrottlerEntry::GetRecommendedDelayForNextRequest() {
+  AutoLock auto_lock(lock_);
+
+  base::TimeTicks now = GetTimeNow();
+  sliding_window_release_time_ =
+      std::max(now, std::max(exponential_backoff_release_time_,
+                             sliding_window_release_time_));
+
+  int64 result = (sliding_window_release_time_ - now).InMillisecondsRoundedUp();
+
+  DCHECK(send_log_.empty() ||
+         sliding_window_release_time_ >= send_log_.back());
+
+  // Log the new send event.
+  send_log_.push(sliding_window_release_time_);
+
+  base::TimeDelta sliding_window_period = base::TimeDelta::FromMilliseconds(
+      sliding_window_period_ms_);
+
+  // Drop the out-of-date events in the event list.
+  // We don't need to worry that the queue may become empty during this
+  // operation, since the last element is sliding_window_release_time_.
+  while ((send_log_.front() + sliding_window_period <=
+          sliding_window_release_time_) ||
+         send_log_.size() > static_cast<unsigned>(max_send_threshold_)) {
+    send_log_.pop();
+  }
+
+  // Check if there are too many send events in recent time.
+  if (send_log_.size() == static_cast<unsigned>(max_send_threshold_))
+    sliding_window_release_time_ = send_log_.front() + sliding_window_period;
+
+  return result;
+}
+
+void RequestThrottlerEntry::UpdateWithResponse(
+    const RequestThrottlerHeaderInterface* response) {
+  AutoLock auto_lock(lock_);
+
+  SaveState();
+  if (response->GetResponseCode() >= 500) {
+    failure_count_++;
+    exponential_backoff_release_time_ =
+        CalculateExponentialBackoffReleaseTime();
+    tracking_exponential_backoff_ = true;
+  } else {
+    // We slowly decay the number of times delayed instead of resetting it to 0
+    // in order to stay stable if we received lots of requests with
+    // malformed bodies at the same time.
+    if (failure_count_ > 0)
+      failure_count_--;
+    tracking_exponential_backoff_ = false;

[Jói, 2010/11/12 00:12:08]

A comment on why this now gets set to false is in order; I believe it's correct
to do this based on how the IsEntryOutdated function works, but it's
non-obvious.

Also, nit: This whole block could use a couple of lines of whitespace.

[yzshen, 2010/11/12 02:05:46]

Since this variable is mainly used for life span management, so I finally decide
to change the comment in IsEntryOutdated(). Please let me know if it looks okay
or not. 
Done.

[Jói, 2010/11/12 19:14:24]

The confusing part is that the invariant of data (according to the name of the
variable, and also the comment you added) that if tracking_exponential_backoff_
is false, then everything is fine - but in fact this variable gets set to false
immediately on a successful request (in UpdateWithResponse) whereas we may still
be "decaying" the number of times delayed - so tracking_exponential_backoff_
being false does not imply failure_count_ == 0.

I think the documentation is probably OK as is though; perhaps in the .h file we
could have a simple invariant for this member, something like:

// If true, the last request response was a failure. Note
// that this member can be false at the same time
// as failure_count_ can be greater than 0.

[yzshen, 2010/11/12 21:28:33]

I could change its name to latest_response_is_error_ and only use it for life
span calculation. I think that is even more clear.

+    // The reason why we are not just cutting the release time to GetTimeNow()
+    // is on the one hand, it would unset delay put by our custom retry-after
+    // header and on the other we would like to push every request up to our
+    // "horizon" when dealing with multiple in-flight requests. Ex: If we send
+    // three requests and we receive 2 failures and 1 success. The success that
+    // follows those failures will not reset the release time, further requests
+    // will then need to wait the delay caused by the 2 failures.
+    exponential_backoff_release_time_ = std::max(
+        GetTimeNow(), exponential_backoff_release_time_);
+    std::string retry_header = response->GetNormalizedValue(kRetryHeaderName);
+    if (!retry_header.empty())
+      HandleCustomRetryAfter(retry_header);
+  }
+}
+
+bool RequestThrottlerEntry::IsEntryOutdated() const {
+  AutoLock auto_lock(lock_);
+
+  if (entry_lifetime_ms_ == -1)
+    return false;
+
+  base::TimeTicks now = GetTimeNow();
+
+  // If there are send events in the sliding window period, we still need this
+  // entry.
+  base::TimeDelta sliding_window_period = base::TimeDelta::FromMilliseconds(
+      sliding_window_period_ms_);
+  if (send_log_.size() > 0 &&
+      send_log_.back() + sliding_window_period > now) {
+    return false;
+  }
+
+  int64 unused_since_ms =
+      (now - exponential_backoff_release_time_).InMilliseconds();
+
+  // Release time is further than now, we are managing it.
+  if (unused_since_ms < 0)
+    return false;
+
+  // There are two cases. First one, when the entry is currently being managed
+  // and should not be collected unless it is older than the maximum allowed
+  // back-off. The other one, when the entry is outdated, unmanaged and should
+  // be collected.
+  if (tracking_exponential_backoff_)
+    return unused_since_ms > std::max(maximum_backoff_ms_, entry_lifetime_ms_);
+
+  return unused_since_ms > entry_lifetime_ms_;
+}
+
+void RequestThrottlerEntry::ReceivedContentWasMalformed() {
+  AutoLock auto_lock(lock_);
+
+  // We should never revert to less back-off or else an attacker could put a
+  // malformed body in cache and replay it to decrease delay.
+  failure_count_ =
+      std::max(old_values_.failure_count, failure_count_);
+  failure_count_++;
+  tracking_exponential_backoff_ = true;
+  exponential_backoff_release_time_ =
+      std::max(CalculateExponentialBackoffReleaseTime(),
+               old_values_.exponential_backoff_release_time);
+}
+
+base::TimeTicks
+    RequestThrottlerEntry::CalculateExponentialBackoffReleaseTime() {
+  lock_.AssertAcquired();
+
+  double delay = initial_backoff_ms_;
+  delay *= pow(multiply_factor_, failure_count_);
+  delay += additional_constant_ms_;
+  delay -= base::RandDouble() * jitter_factor_ * delay;
+
+  // Ensure that we do not exceed maximum delay.
+  int64 delay_int = static_cast<int64>(delay + 0.5);
+  delay_int = std::min(delay_int, static_cast<int64>(maximum_backoff_ms_));
+
+  return std::max(GetTimeNow() + base::TimeDelta::FromMilliseconds(delay_int),
+                  exponential_backoff_release_time_);
+}
+
+base::TimeTicks RequestThrottlerEntry::GetTimeNow() const {
+  return base::TimeTicks::Now();
+}
+
+void RequestThrottlerEntry::HandleCustomRetryAfter(
+    const std::string& header_value) {
+  lock_.AssertAcquired();
+
+  // Input parameter is the number of seconds to wait in a floating point value.
+  double time_in_sec = 0;
+  bool conversion_is_ok = base::StringToDouble(header_value, &time_in_sec);
+
+  // Conversion of custom retry-after header value failed.
+  if (!conversion_is_ok)
+    return;
+
+  // We must use an int value later so we transform this in milliseconds.
+  int64 value_ms = static_cast<int64>(0.5 + time_in_sec * 1000);
+
+  if (maximum_backoff_ms_ < value_ms || value_ms < 0)
+    return;
+
+  exponential_backoff_release_time_ = std::max(
+      (GetTimeNow() + base::TimeDelta::FromMilliseconds(value_ms)),
+      exponential_backoff_release_time_);
+}
+
+void RequestThrottlerEntry::SaveState() {
+  lock_.AssertAcquired();
+
+  old_values_.exponential_backoff_release_time =
+      exponential_backoff_release_time_;
+  old_values_.failure_count = failure_count_;
+}