Implement exponential back-off mechanism and enforce it at the URLRequestHttpJob level.

net/request_throttler/request_throttler_entry.cc

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/request_throttler/request_throttler_entry.h"
+
+#include <cmath>
+
+#include "base/logging.h"
+#include "base/rand_util.h"
+#include "base/string_number_conversions.h"
+#include "net/request_throttler/request_throttler_header_interface.h"
+
+const int RequestThrottlerEntry::kDefaultSlidingWindowPeriodMs = 2000;
+const int RequestThrottlerEntry::kDefaultMaxSendThreshold = 20;
+const int RequestThrottlerEntry::kDefaultInitialBackoffMs = 700;
+const int RequestThrottlerEntry::kDefaultAdditionalConstantMs = 100;
+const double RequestThrottlerEntry::kDefaultMultiplyFactor = 2.0;
+const double RequestThrottlerEntry::kDefaultJitterFactor = 0.4;
+const int RequestThrottlerEntry::kDefaultMaximumBackoffMs = 24 * 60 * 60 * 1000;
+const int RequestThrottlerEntry::kDefaultEntryLifetimeMs = 120000;
+const char RequestThrottlerEntry::kRetryHeaderName[] = "X-Retry-After";
+
+RequestThrottlerEntry::RequestThrottlerEntry()
+    : sliding_window_period_ms_(kDefaultSlidingWindowPeriodMs),
+      max_send_threshold_(kDefaultMaxSendThreshold),
+      initial_backoff_ms_(kDefaultInitialBackoffMs),
+      additional_constant_ms_(kDefaultAdditionalConstantMs),
+      multiply_factor_(kDefaultMultiplyFactor),
+      jitter_factor_(kDefaultJitterFactor),
+      maximum_backoff_ms_(kDefaultMaximumBackoffMs),
+      entry_lifetime_ms_(kDefaultEntryLifetimeMs) {
+  Initialize();
+}
+
+RequestThrottlerEntry::RequestThrottlerEntry(
+    int sliding_window_period_ms,
+    int max_send_threshold,
+    int initial_backoff_ms,
+    int additional_constant_ms,
+    double multiply_factor,
+    double jitter_factor,
+    int maximum_backoff_ms,
+    int entry_lifetime_ms)
+    : sliding_window_period_ms_(sliding_window_period_ms),
+      max_send_threshold_(max_send_threshold),
+      initial_backoff_ms_(initial_backoff_ms),
+      additional_constant_ms_(additional_constant_ms),
+      multiply_factor_(multiply_factor),
+      jitter_factor_(jitter_factor),
+      maximum_backoff_ms_(maximum_backoff_ms),
+      entry_lifetime_ms_(entry_lifetime_ms) {
+  DCHECK(sliding_window_period_ms_ > 0 &&
+         max_send_threshold_ > 0 &&
+         initial_backoff_ms_ >= 0 &&
+         additional_constant_ms_ >= 0 &&
+         multiply_factor_ > 0 &&
+         jitter_factor_ >= 0 &&
+         maximum_backoff_ms_ >= 0 &&
+         entry_lifetime_ms_ > 0);
+
+  Initialize();
+}
+
+RequestThrottlerEntry::~RequestThrottlerEntry() {
+}
+
+void RequestThrottlerEntry::Initialize() {
+  release_time_ = base::TimeTicks::Now();
+  num_times_delayed_ = 0;
+  is_managed_ = false;
+
+  old_values_.release_time = release_time_;
+  old_values_.number_of_failed_requests = num_times_delayed_;
+}
+
+bool RequestThrottlerEntry::IsRequestAllowed() const {
+  AutoLock auto_lock(lock_);
+  return release_time_ <= GetTimeNow();
+}
+
+void RequestThrottlerEntry::UpdateWithResponse(
+    const RequestThrottlerHeaderInterface* response) {
+  AutoLock auto_lock(lock_);
+
+  SaveState();
+  if (response->GetResponseCode() >= 500) {
+    num_times_delayed_++;
+    release_time_ = std::max(CalculateReleaseTime(), release_time_);
+    is_managed_ = true;
+  } else {
+    // We slowly decay the number of times delayed instead of resetting it to 0
+    // in order to stay stable if we received lots of requests with
+    // malformed bodies at the same time.
+    if (num_times_delayed_ > 0)
+      num_times_delayed_--;
+    is_managed_ = false;
+    // The reason why we are not just cutting release_time to GetTimeNow() is
+    // on the one hand, it would unset delay put by our custom retry-after
+    // header and on the other we would like to push every request up to our
+    // "horizon" when dealing with multiple in-flight request. Ex: If we send
+    // three request and we receive 2 failures and 1 success. The success that
+    // follows those failures will not reset release time further request will
+    // then need to wait the delay caused by the 2 failures.
+    release_time_ = std::max(GetTimeNow(), release_time_);
+    std::string retry_header = response->GetNormalizedValue(kRetryHeaderName);
+    if (!retry_header.empty())
+      HandleCustomRetryAfter(retry_header);
+  }
+}
+
+void RequestThrottlerEntry::NotifyRequestStart() {
+  AutoLock auto_lock(lock_);
+
+  base::TimeDelta sliding_window_period = base::TimeDelta::FromMilliseconds(
+      sliding_window_period_ms_);
+  release_time_ = std::max(release_time_, GetTimeNow());
+  if (send_log_.size() > 0)
+    release_time_ = std::max(release_time_, send_log_.back());
+
+  // Log the new send event.
+  send_log_.push(release_time_);
+
+  // Drop the out-of-date events in the event list.
+  // We don't need to worry that the queue may become empty during this
+  // operation, since the last element is release_time_.
+  while (send_log_.front() + sliding_window_period <= release_time_) {
+    send_log_.pop();
+  }
+
+  // Check if there are too many send events in recent time.
+  if (send_log_.size() >= static_cast<unsigned>(max_send_threshold_))
+    release_time_ = send_log_.front() + sliding_window_period;
+}
+
+bool RequestThrottlerEntry::IsEntryOutdated() const {
+  AutoLock auto_lock(lock_);
+
+  base::TimeTicks now = GetTimeNow();
+  int64 unused_since_ms = (now - release_time_).InMilliseconds();
+
+  // Release time is further than now, we are managing it.
+  if (unused_since_ms < 0)
+    return false;
+
+  // If there are send events in the sliding window period, we still need this
+  // entry.
+  base::TimeDelta sliding_window_period = base::TimeDelta::FromMilliseconds(
+      sliding_window_period_ms_);
+  if (send_log_.size() > 0 &&
+      send_log_.back() + sliding_window_period > now) {
+    return false;
+  }
+
+  // There are two cases. First one, when the entry is currently being managed
+  // and should not be collected unless it is older than the maximum allowed
+  // back-off. The other one, when the entry is outdated, unmanaged and should
+  // be collected.
+  if (is_managed_)
+    return unused_since_ms > std::max(maximum_backoff_ms_, entry_lifetime_ms_);
+
+  return unused_since_ms > entry_lifetime_ms_;
+}
+
+void RequestThrottlerEntry::ReceivedContentWasMalformed() {
+  AutoLock auto_lock(lock_);
+
+  // We should never revert to less back-off or else an attacker could put a
+  // malformed body in cache and replay it to decrease delay.
+  num_times_delayed_ =
+      std::max(old_values_.number_of_failed_requests, num_times_delayed_);
+  num_times_delayed_++;
+  is_managed_ = true;
+  release_time_ = std::max(CalculateReleaseTime(),
+                           std::max(old_values_.release_time, release_time_));
+}
+
+base::TimeTicks RequestThrottlerEntry::release_time() const {
+  AutoLock auto_lock(lock_);
+  return release_time_;
+}
+
+base::TimeTicks RequestThrottlerEntry::CalculateReleaseTime() {
+  lock_.AssertAcquired();
+
+  double delay = initial_backoff_ms_;
+  delay *= pow(multiply_factor_, num_times_delayed_);
+  delay += additional_constant_ms_;
+  delay -= base::RandDouble() * jitter_factor_ * delay;
+
+  // Ensure that we do not exceed maximum delay.
+  int64 delay_int = static_cast<int64>(delay + 0.5);
+  delay_int = std::min(delay_int, static_cast<int64>(maximum_backoff_ms_));
+
+  return GetTimeNow() + base::TimeDelta::FromMilliseconds(delay_int);
+}
+
+base::TimeTicks RequestThrottlerEntry::GetTimeNow() const {
+  return base::TimeTicks::Now();
+}
+
+void RequestThrottlerEntry::HandleCustomRetryAfter(
+    const std::string& header_value) {
+  lock_.AssertAcquired();
+
+  // Input parameter is the number of seconds to wait in a floating point value.
+  double time_in_sec = 0;
+  bool conversion_is_ok = base::StringToDouble(header_value, &time_in_sec);
+
+  // Conversion of custom retry-after header value failed.
+  if (!conversion_is_ok)
+    return;
+
+  // We must use an int value later so we transform this in milliseconds.
+  int64 value_ms = static_cast<int64>(0.5 + time_in_sec * 1000);
+
+  if (maximum_backoff_ms_ < value_ms || value_ms < 0)
+    return;
+
+  release_time_ = std::max(
+      (GetTimeNow() + base::TimeDelta::FromMilliseconds(value_ms)),
+      release_time_);
+}
+
+void RequestThrottlerEntry::SaveState() {
+  lock_.AssertAcquired();
+
+  old_values_.release_time = release_time_;
+  old_values_.number_of_failed_requests = num_times_delayed_;
+}