Add MediaServiceThrottler

media/base/android/media_service_throttler.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/base/android/media_service_throttler.h"
+
+#include "base/bind.h"
+#include "base/memory/ptr_util.h"
+#include "base/threading/thread_task_runner_handle.h"
+#include "base/time/default_tick_clock.h"
+#include "media/base/android/media_server_crash_listener.h"
+
+namespace {

[sandersd (OOO until July 31), 2016/11/11 18:55:14]

Nit: The anonymous namespace is inside the named namespace in the majority of
Chromium code.

[tguilbert, 2016/11/14 23:18:08]

Done.

+
+static base::LazyInstance<media::MediaServiceThrottler>::Leaky
+    g_media_service_throttler = LAZY_INSTANCE_INITIALIZER;
+
+// Elapsed time between crashes needed to completely reset the media server
+// crash count.
+constexpr base::TimeDelta kTimeUntilCrashReset =
+    base::TimeDelta::FromMinutes(1);
+
+// Elapsed time between schedule calls needed to completely reset the
+// scheduling clock.
+constexpr base::TimeDelta kTimeUntilScheduleReset =
+    base::TimeDelta::FromMinutes(1);
+
+// Decay rate of server crashes, corresponding to a tolerable 'normal' crash
+// rate. This means that we will decrement our crash rate by ~1 crash/minute.
+const uint32_t kCrashDecayPeriodInMs = 60000;
+
+// Rate at which client creations will be exponentially throttled based on the
+// number of media server crashes.
+// NOTE: Since our exponential delay formula is 2^(server crashes), 0 server
+// crashes still result in this delay being added once.
+constexpr base::TimeDelta kBaseExponentialDelay =
+    base::TimeDelta::FromMilliseconds(120);
+
+// Base rate at which we schedule client creations.
+// The minimal delay is |kLinearThrottlingDelay| + |kBaseExponentialDelay|.
+// This corresponds to 0.2s.
+constexpr base::TimeDelta kLinearThrottlingDelay =
+    base::TimeDelta::FromMilliseconds(80);
+
+// Max exponential throttling rate from media server crashes.
+// The max delay will still be |kLinearThrottlingDelay| +
+// |kMaxExponentialDelay|. This corresponds to 3s.
+constexpr base::TimeDelta kMaxExponentialDelay =
+    base::TimeDelta::FromMilliseconds(2920);
+
+// Max number of clients to schedule immediately (e.g when loading a new page).
+const uint32_t kMaxBurstClients = 10;
+
+// Sliding window of time during which we allow clients to be scheduled
+// immediately, to accomodate for a "bursts" of requests when loading new pages.
+const base::TimeDelta kMinDelayWindow =
+    (kLinearThrottlingDelay + kBaseExponentialDelay) * kMaxBurstClients;
+
+// The throttling progression based on number of crashes looks as follows:
+//
+// | # crashes | period  | clients/sec | clients/mins | # burst clients
+// | 0         | 200  ms | 5.0         | 300          | 10
+// | 1         | 320  ms | 3.1         | 188          | 6
+// | 2         | 560  ms | 1.8         | 107          | 4
+// | 3         | 1040 ms | 1.0         | 58           | 2
+// | 4         | 2000 ms | 0.5         | 30           | 1
+// | 5         | 3000 ms | 0.3         | 20           | 1
+// | 6         | 3000 ms | 0.3         | 20           | 1
+//
+// NOTE: Since we use the floor function and a decay rate of 1 crash/minute when
+// calculating the effective # of crashes, a single crash per minute will result
+// in 0 effective crashes (since floor(1.0 - 'tiny decay') is 0). If we
+// experience slightly more than 1 crash per 60 seconds, the effective number of
+// crashes will go up as expected.
+}
+
+namespace media {
+
+// static
+MediaServiceThrottler* MediaServiceThrottler::GetInstance() {
+  return g_media_service_throttler.Pointer();
+}
+
+MediaServiceThrottler::~MediaServiceThrottler() {}
+
+MediaServiceThrottler::MediaServiceThrottler()
+    : clock_(new base::DefaultTickClock()),
+      current_crashes_(0),
+      crash_listener_task_runner_(base::ThreadTaskRunnerHandle::Get()) {
+  // base::Unretained is safe because the MediaServiceThrottler is supposed to
+  // live until the process dies.
+  release_crash_listener_cb_ = base::Bind(
+      &MediaServiceThrottler::ReleaseCrashListener, base::Unretained(this));
+  EnsureCrashListenerStarted();
+}
+
+void MediaServiceThrottler::SetTickClockForTesting(base::TickClock* clock) {
+  clock_.reset(clock);
+}
+
+base::TimeDelta MediaServiceThrottler::GetBaseThrottlingRateForTesting() {
+  return kBaseExponentialDelay + kLinearThrottlingDelay;
+}
+
+void MediaServiceThrottler::ResetInternalStateForTesting() {
+  last_server_crash_ = base::TimeTicks();
+  last_schedule_call_ = base::TimeTicks();
+  next_schedulable_slot_ = clock_->NowTicks();
+  last_current_crash_update_time_ = clock_->NowTicks();
+  current_crashes_ = 0.0;
+}
+
+base::TimeDelta MediaServiceThrottler::GetDelayForClientCreation() {
+  // Make sure the listener is started and the crashes decayed.
+  EnsureCrashListenerStarted();
+  UpdateServerCrashes();
+
+  base::TimeTicks now = clock_->NowTicks();
+
+  // If we are passed the next time slot or if it has been 1 minute since the
+  // last call to GetDelayForClientCreation(), reset the next time to now.
+  if (now > next_schedulable_slot_ ||
+      (now - last_schedule_call_) > kTimeUntilScheduleReset) {
+    next_schedulable_slot_ = now;
+  }
+
+  last_schedule_call_ = now;
+
+  // Increment the next scheduled time between 0.2s and 3s, which allows the
+  // creation of between 50 and 3 clients per 10s.
+  next_schedulable_slot_ +=
+      kLinearThrottlingDelay + GetThrottlingDelayFromServerCrashes();
+
+  // Calculate how long to delay the creation so it isn't scheduled before
+  // |next_schedulable_slot_|.
+  base::TimeDelta delay = next_schedulable_slot_ - now;
+
+  // If the scheduling delay is low enough, schedule it immediately instead.
+  // This allows up to kMaxBurstClients clients to be scheduled immediately.
+  if (delay <= kMinDelayWindow)
+    return base::TimeDelta();
+
+  return delay;
+}
+
+base::TimeDelta MediaServiceThrottler::GetThrottlingDelayFromServerCrashes() {
+  // The combination of rounding down the number of crashes down and decaying
+  // at the rate of 1 crash / min means that a single crash will very quickly be
+  // rounded down to 0. Effectively, this means that we only start exponentially
+  // backing off if we have more than 1 crash in a 60 second window.
+  uint32_t number_crashes = std::floor(current_crashes_);

[watk, 2016/11/12 04:22:16]

number_crashes sounds weird but num_crashes doesn't. Weird.

[tguilbert, 2016/11/14 23:18:08]

Done.

+  DCHECK_GE(number_crashes, 0u);
+
+  // Prevents overflow/undefined behavior. We already reach kMaxExponentialDelay
+  // at 5 crashes in any case.
+  number_crashes = std::min(number_crashes, 10u);
+
+  return std::min(kBaseExponentialDelay * (1 << number_crashes),
+                  kMaxExponentialDelay);
+}
+
+void MediaServiceThrottler::OnMediaServerCrash() {
+  UpdateServerCrashes();
+
+  last_server_crash_ = clock_->NowTicks();
+  current_crashes_ += 1.0;
+}
+
+void MediaServiceThrottler::UpdateServerCrashes() {
+  base::TimeTicks now = clock_->NowTicks();
+  base::TimeDelta time_since_last_crash = now - last_server_crash_;
+
+  if (time_since_last_crash > kTimeUntilCrashReset) {

[watk, 2016/11/12 04:22:16]

Sorry if this is a dumb question, but is this special case necessary? Seems
handled by the else.

[tguilbert, 2016/11/14 23:18:08]

Not a dumb question :)

We decay at a rate of 1/min, but crashes can occur faster than 1/min. If we are
crashing 3 times per min, |current_crashes| will go up at the effective rate of
2/min, and we will never enter the if.

The 'if' portion is to completely reset the count if we did not crash for a
minute. Otherwise, we might end up with a burst of crashes, bringing
|current_crashes_| to 60, and it would take an hour to stop throttling.

+    // Reset the number of crashes if we haven't had a crash in the past minute.
+    current_crashes_ = 0.0;
+  } else {
+    // Decay at the rate of 1 crash/minute otherwise.
+    double decay = (now - last_current_crash_update_time_).InMillisecondsF() /
+                   kCrashDecayPeriodInMs;
+    current_crashes_ = std::max(0.0, current_crashes_ - decay);
+  }
+
+  last_current_crash_update_time_ = now;
+}
+
+void MediaServiceThrottler::ReleaseCrashListener() {
+  crash_listener_.reset(nullptr);
+}
+
+void MediaServiceThrottler::EnsureCrashListenerStarted() {
+  if (!crash_listener_) {
+    // base::Unretained is safe here because both the MediaServiceThrottler and
+    // the MediaServerCrashListener live until the process is terminated.
+    crash_listener_ = base::MakeUnique<MediaServerCrashListener>(
+        base::Bind(&MediaServiceThrottler::OnMediaServerCrash,
+                   base::Unretained(this)),
+        crash_listener_task_runner_);
+  } else {
+    crash_listener_->EnsureListening();
+  }
+
+  // Cancels outstanding/pending versions of the callback.
+  cancelable_release_crash_listener_cb_.Reset(release_crash_listener_cb_);
+
+  crash_listener_task_runner_->PostDelayedTask(
+      FROM_HERE, cancelable_release_crash_listener_cb_.callback(),
+      base::TimeDelta::FromMinutes(1));

[DaleCurtis, 2016/11/11 23:32:16]

This should probably be more clearly related to your decay constants above.

[tguilbert, 2016/11/14 23:18:08]

Added a new constant and comments.

+}
+
+bool MediaServiceThrottler::IsCrashListenerAliveForTesting() {
+  return !!crash_listener_;
+}
+
+void MediaServiceThrottler::SetCrashListenerTaskRunnerForTesting(
+    scoped_refptr<base::SingleThreadTaskRunner> crash_listener_task_runner) {
+  // Set the task runner so |crash_listener_| be deleted on the right thread.
+  crash_listener_task_runner_ = crash_listener_task_runner;
+
+  // Re-create the crash listener.
+  crash_listener_ = base::MakeUnique<MediaServerCrashListener>(
+      base::Closure(), crash_listener_task_runner_);
+}
+
+}  // namespace media