[Sync] Sync client should to exponential backoff when receive partial failure

components/sync/engine_impl/sync_scheduler_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/sync/engine_impl/sync_scheduler_impl.h"
 
 #include <algorithm>
 #include <cstring>
 #include <utility>
 
@@ skipping 70 matching lines @@
 }
 
 void RunAndReset(base::Closure* task) {
   DCHECK(task);
   if (task->is_null())
     return;
   task->Run();
   task->Reset();
 }
 
+#define ENUM_CASE(x) \
+  case x:            \
+    return #x;       \
+    break;
+
 }  // namespace
 
 ConfigurationParams::ConfigurationParams()
     : source(GetUpdatesCallerInfo::UNKNOWN) {}
 ConfigurationParams::ConfigurationParams(
     const sync_pb::GetUpdatesCallerInfo::GetUpdatesSource& source,
     ModelTypeSet types_to_download,
     const ModelSafeRoutingInfo& routing_info,
     const base::Closure& ready_task,
     const base::Closure& retry_task)
     : source(source),
       types_to_download(types_to_download),
       routing_info(routing_info),
       ready_task(ready_task),
       retry_task(retry_task) {
   DCHECK(!ready_task.is_null());
 }
 ConfigurationParams::ConfigurationParams(const ConfigurationParams& other) =
     default;
 ConfigurationParams::~ConfigurationParams() {}
 
 ClearParams::ClearParams(const base::Closure& report_success_task)
     : report_success_task(report_success_task) {
   DCHECK(!report_success_task.is_null());
 }
 ClearParams::ClearParams(const ClearParams& other) = default;
 ClearParams::~ClearParams() {}
 
-SyncSchedulerImpl::WaitInterval::WaitInterval() : mode(UNKNOWN) {}
-
-SyncSchedulerImpl::WaitInterval::WaitInterval(Mode mode, TimeDelta length)
-    : mode(mode), length(length) {}
-
-SyncSchedulerImpl::WaitInterval::~WaitInterval() {}
-
-#define ENUM_CASE(x) \
-  case x:            \
-    return #x;       \
-    break;
-
-const char* SyncSchedulerImpl::WaitInterval::GetModeString(Mode mode) {
-  switch (mode) {
-    ENUM_CASE(UNKNOWN);
-    ENUM_CASE(EXPONENTIAL_BACKOFF);
-    ENUM_CASE(THROTTLED);
-  }
-  NOTREACHED();
-  return "";
-}
-
 GetUpdatesCallerInfo::GetUpdatesSource GetUpdatesFromNudgeSource(
     NudgeSource source) {
   switch (source) {
     case NUDGE_SOURCE_NOTIFICATION:
       return GetUpdatesCallerInfo::NOTIFICATION;
     case NUDGE_SOURCE_LOCAL:
       return GetUpdatesCallerInfo::LOCAL;
     case NUDGE_SOURCE_LOCAL_REFRESH:
       return GetUpdatesCallerInfo::DATATYPE_REFRESH;
     case NUDGE_SOURCE_UNKNOWN:
@@ skipping 93 matching lines @@
   Mode old_mode = mode_;
   mode_ = mode;
   // Only adjust the poll reset time if it was valid and in the past.
   if (!last_poll_time.is_null() && last_poll_time < base::Time::Now()) {
     // Convert from base::Time to base::TimeTicks. The reason we use Time
     // for persisting is that TimeTicks can stop making forward progress when
     // the machine is suspended. This implies that on resume the client might
     // actually have miss the real poll, unless the client is restarted. Fixing
     // that would require using an AlarmTimer though, which is only supported
     // on certain platforms.
-    last_poll_reset_ =
-        base::TimeTicks::Now() - (base::Time::Now() - last_poll_time);
+    last_poll_reset_ = TimeTicks::Now() - (base::Time::Now() - last_poll_time);
   }
 
   if (old_mode != mode_ && mode_ == NORMAL_MODE) {
     // We just got back to normal mode.  Let's try to run the work that was
     // queued up while we were configuring.
 
     AdjustPolling(UPDATE_INTERVAL);  // Will kick start poll timer if needed.
 
     // Update our current time before checking IsRetryRequired().
-    nudge_tracker_.SetSyncCycleStartTime(base::TimeTicks::Now());
+    nudge_tracker_.SetSyncCycleStartTime(TimeTicks::Now());
     if (nudge_tracker_.IsSyncRequired() && CanRunNudgeJobNow(NORMAL_PRIORITY)) {
       TrySyncCycleJob();
     }
   }
 }
 
-ModelTypeSet SyncSchedulerImpl::GetEnabledAndUnthrottledTypes() {
+ModelTypeSet SyncSchedulerImpl::GetEnabledAndUnblockedTypes() {
   ModelTypeSet enabled_types = cycle_context_->GetEnabledTypes();
   ModelTypeSet enabled_protocol_types =
       Intersection(ProtocolTypes(), enabled_types);
-  ModelTypeSet throttled_types = nudge_tracker_.GetThrottledTypes();
-  return Difference(enabled_protocol_types, throttled_types);
+  ModelTypeSet blocked_types = nudge_tracker_.GetBlockedTypes();
+  return Difference(enabled_protocol_types, blocked_types);
 }
 
 void SyncSchedulerImpl::SendInitialSnapshot() {
   DCHECK(CalledOnValidThread());
   std::unique_ptr<SyncCycle> dummy(SyncCycle::Build(cycle_context_, this));
   SyncCycleEvent event(SyncCycleEvent::STATUS_CHANGED);
   event.snapshot = dummy->TakeSnapshot();
   for (auto& observer : *cycle_context_->listeners())
     observer.OnSyncCycleEvent(event);
 }
@@ skipping 77 matching lines @@
 
 bool SyncSchedulerImpl::CanRunNudgeJobNow(JobPriority priority) {
   DCHECK(CalledOnValidThread());
 
   if (!CanRunJobNow(priority)) {
     SDVLOG(1) << "Unable to run a nudge job right now";
     return false;
   }
 
   const ModelTypeSet enabled_types = cycle_context_->GetEnabledTypes();
-  if (nudge_tracker_.GetThrottledTypes().HasAll(enabled_types)) {
-    SDVLOG(1) << "Not running a nudge because we're fully type throttled.";
+  if (nudge_tracker_.GetBlockedTypes().HasAll(enabled_types)) {
+    SDVLOG(1) << "Not running a nudge because we're fully type throttled or "
+                 "backed off.";
     return false;
   }
 
   if (mode_ != NORMAL_MODE) {
     SDVLOG(1) << "Not running nudge because we're not in normal mode.";
     return false;
   }
 
   return true;
 }
 
 void SyncSchedulerImpl::ScheduleLocalNudge(
     ModelTypeSet types,
     const tracked_objects::Location& nudge_location) {
   DCHECK(CalledOnValidThread());
   DCHECK(!types.Empty());
 
   SDVLOG_LOC(nudge_location, 2) << "Scheduling sync because of local change to "
                                 << ModelTypeSetToString(types);
   UpdateNudgeTimeRecords(types);
-  base::TimeDelta nudge_delay = nudge_tracker_.RecordLocalChange(types);
+  TimeDelta nudge_delay = nudge_tracker_.RecordLocalChange(types);
   ScheduleNudgeImpl(nudge_delay, nudge_location);
 }
 
 void SyncSchedulerImpl::ScheduleLocalRefreshRequest(
     ModelTypeSet types,
     const tracked_objects::Location& nudge_location) {
   DCHECK(CalledOnValidThread());
   DCHECK(!types.Empty());
 
   SDVLOG_LOC(nudge_location, 2)
       << "Scheduling sync because of local refresh request for "
       << ModelTypeSetToString(types);
-  base::TimeDelta nudge_delay = nudge_tracker_.RecordLocalRefreshRequest(types);
+  TimeDelta nudge_delay = nudge_tracker_.RecordLocalRefreshRequest(types);
   ScheduleNudgeImpl(nudge_delay, nudge_location);
 }
 
 void SyncSchedulerImpl::ScheduleInvalidationNudge(
     ModelType model_type,
     std::unique_ptr<InvalidationInterface> invalidation,
     const tracked_objects::Location& nudge_location) {
   DCHECK(CalledOnValidThread());
 
   SDVLOG_LOC(nudge_location, 2)
       << "Scheduling sync because we received invalidation for "
       << ModelTypeToString(model_type);
-  base::TimeDelta nudge_delay = nudge_tracker_.RecordRemoteInvalidation(
+  TimeDelta nudge_delay = nudge_tracker_.RecordRemoteInvalidation(
       model_type, std::move(invalidation));
   ScheduleNudgeImpl(nudge_delay, nudge_location);
 }
 
 void SyncSchedulerImpl::ScheduleInitialSyncNudge(ModelType model_type) {
   DCHECK(CalledOnValidThread());
 
   SDVLOG(2) << "Scheduling non-blocking initial sync for "
             << ModelTypeToString(model_type);
   nudge_tracker_.RecordInitialSyncRequired(model_type);
@@ skipping 14 matching lines @@
     return;
   }
 
   SDVLOG_LOC(nudge_location, 2) << "In ScheduleNudgeImpl with delay "
                                 << delay.InMilliseconds() << " ms";
 
   if (!CanRunNudgeJobNow(NORMAL_PRIORITY))
     return;
 
   TimeTicks incoming_run_time = TimeTicks::Now() + delay;
-  if (!scheduled_nudge_time_.is_null() &&
-      (scheduled_nudge_time_ < incoming_run_time)) {
+  if (pending_wakeup_timer_.IsRunning() &&
+      (pending_wakeup_timer_.desired_run_time() < incoming_run_time)) {
     // Old job arrives sooner than this one.  Don't reschedule it.
     return;
   }
 
   // Either there is no existing nudge in flight or the incoming nudge should be
   // made to arrive first (preempt) the existing nudge.  We reschedule in either
   // case.
   SDVLOG_LOC(nudge_location, 2) << "Scheduling a nudge with "
                                 << delay.InMilliseconds() << " ms delay";
-  scheduled_nudge_time_ = incoming_run_time;
   pending_wakeup_timer_.Start(
       nudge_location, delay, base::Bind(&SyncSchedulerImpl::PerformDelayedNudge,
                                         weak_ptr_factory_.GetWeakPtr()));
 }
 
 const char* SyncSchedulerImpl::GetModeString(SyncScheduler::Mode mode) {
   switch (mode) {
     ENUM_CASE(CONFIGURATION_MODE);
     ENUM_CASE(CLEAR_SERVER_DATA_MODE);
     ENUM_CASE(NORMAL_MODE);
   }
   return "";
 }
 
-void SyncSchedulerImpl::SetDefaultNudgeDelay(base::TimeDelta delay_ms) {
+void SyncSchedulerImpl::SetDefaultNudgeDelay(TimeDelta delay_ms) {
   DCHECK(CalledOnValidThread());
   nudge_tracker_.SetDefaultNudgeDelay(delay_ms);
 }
 
 void SyncSchedulerImpl::DoNudgeSyncCycleJob(JobPriority priority) {
   DCHECK(CalledOnValidThread());
   DCHECK(CanRunNudgeJobNow(priority));
 
   DVLOG(2) << "Will run normal mode sync cycle with types "
            << ModelTypeSetToString(cycle_context_->GetEnabledTypes());
   std::unique_ptr<SyncCycle> cycle(SyncCycle::Build(cycle_context_, this));
-  bool success = syncer_->NormalSyncShare(GetEnabledAndUnthrottledTypes(),
+  bool success = syncer_->NormalSyncShare(GetEnabledAndUnblockedTypes(),
                                           &nudge_tracker_, cycle.get());
 
   if (success) {
     // That cycle took care of any outstanding work we had.
     SDVLOG(2) << "Nudge succeeded.";
     nudge_tracker_.RecordSuccessfulSyncCycle();
-    scheduled_nudge_time_ = base::TimeTicks();
     HandleSuccess();
 
     // If this was a canary, we may need to restart the poll timer (the poll
     // timer may have fired while the scheduler was in an error state, ignoring
     // the poll).
     if (!poll_timer_.IsRunning()) {
       SDVLOG(1) << "Canary succeeded, restarting polling.";
       AdjustPolling(UPDATE_INTERVAL);
     }
   } else {
@@ skipping 80 matching lines @@
     wait_interval_ = base::MakeUnique<WaitInterval>(
         WaitInterval::EXPONENTIAL_BACKOFF, length);
     SDVLOG(2) << "Sync cycle failed.  Will back off for "
               << wait_interval_->length.InMilliseconds() << "ms.";
   }
   RestartWaiting();
 }
 
 void SyncSchedulerImpl::DoPollSyncCycleJob() {
   SDVLOG(2) << "Polling with types "
-            << ModelTypeSetToString(GetEnabledAndUnthrottledTypes());
+            << ModelTypeSetToString(GetEnabledAndUnblockedTypes());
   std::unique_ptr<SyncCycle> cycle(SyncCycle::Build(cycle_context_, this));
   bool success =
-      syncer_->PollSyncShare(GetEnabledAndUnthrottledTypes(), cycle.get());
+      syncer_->PollSyncShare(GetEnabledAndUnblockedTypes(), cycle.get());
 
   // Only restart the timer if the poll succeeded. Otherwise rely on normal
   // failure handling to retry with backoff.
   if (success) {
     AdjustPolling(FORCE_RESET);
     HandleSuccess();
   } else {
     HandleFailure(cycle->status_controller().model_neutral_state());
   }
 }
 
 void SyncSchedulerImpl::UpdateNudgeTimeRecords(ModelTypeSet types) {
   DCHECK(CalledOnValidThread());
-  base::TimeTicks now = TimeTicks::Now();
+  TimeTicks now = TimeTicks::Now();
   // Update timing information for how often datatypes are triggering nudges.
   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
-    base::TimeTicks previous = last_local_nudges_by_model_type_[iter.Get()];
+    TimeTicks previous = last_local_nudges_by_model_type_[iter.Get()];
     last_local_nudges_by_model_type_[iter.Get()] = now;
     if (previous.is_null())
       continue;
 
 #define PER_DATA_TYPE_MACRO(type_str) \
   SYNC_FREQ_HISTOGRAM("Sync.Freq" type_str, now - previous);
     SYNC_DATA_TYPE_HISTOGRAM(iter.Get());
 #undef PER_DATA_TYPE_MACRO
   }
 }
@@ skipping 42 matching lines @@
 
   SDVLOG(1) << "Updating polling delay to " << poll_delay.InMinutes()
             << " minutes.";
 
   // Adjust poll rate. Start will reset the timer if it was already running.
   poll_timer_.Start(FROM_HERE, poll_delay, this,
                     &SyncSchedulerImpl::PollTimerCallback);
 }
 
 void SyncSchedulerImpl::RestartWaiting() {
-  CHECK(wait_interval_.get());
-  DCHECK(wait_interval_->length >= TimeDelta::FromSeconds(0));
-  NotifyRetryTime(base::Time::Now() + wait_interval_->length);
-  SDVLOG(2) << "Starting WaitInterval timer of length "
-            << wait_interval_->length.InMilliseconds() << "ms.";
-  if (wait_interval_->mode == WaitInterval::THROTTLED) {
-    pending_wakeup_timer_.Start(FROM_HERE, wait_interval_->length,
-                                base::Bind(&SyncSchedulerImpl::Unthrottle,
+  if (wait_interval_.get()) {
+    // Global throttling or backoff
+    NotifyRetryTime(base::Time::Now() + wait_interval_->length);
+    SDVLOG(2) << "Starting WaitInterval timer of length "
+              << wait_interval_->length.InMilliseconds() << "ms.";
+    if (wait_interval_->mode == WaitInterval::THROTTLED) {
+      pending_wakeup_timer_.Start(FROM_HERE, wait_interval_->length,
+                                  base::Bind(&SyncSchedulerImpl::Unthrottle,
+                                             weak_ptr_factory_.GetWeakPtr()));
+    } else {
+      pending_wakeup_timer_.Start(
+          FROM_HERE, wait_interval_->length,
+          base::Bind(&SyncSchedulerImpl::ExponentialBackoffRetry,
+                     weak_ptr_factory_.GetWeakPtr()));
+    }
+  } else if (nudge_tracker_.IsAnyTypeBlocked()) {
+    // Per-datatype throttled or backed off.
+    TimeDelta time_until_next_unblock =
+        nudge_tracker_.GetTimeUntilNextUnblock();
+    pending_wakeup_timer_.Start(FROM_HERE, time_until_next_unblock,
+                                base::Bind(&SyncSchedulerImpl::OnTypesUnblocked,
                                            weak_ptr_factory_.GetWeakPtr()));
-  } else {
-    pending_wakeup_timer_.Start(
-        FROM_HERE, wait_interval_->length,
-        base::Bind(&SyncSchedulerImpl::ExponentialBackoffRetry,
-                   weak_ptr_factory_.GetWeakPtr()));
   }
 }
 
 void SyncSchedulerImpl::Stop() {
   DCHECK(CalledOnValidThread());
   SDVLOG(2) << "Stop called";
 
   // Kill any in-flight method calls.
   weak_ptr_factory_.InvalidateWeakPtrs();
   wait_interval_.reset();
@@ skipping 19 matching lines @@
   // access token will be here.
   base::ThreadTaskRunnerHandle::Get()->PostTask(
       FROM_HERE, base::Bind(&SyncSchedulerImpl::TrySyncCycleJobImpl,
                             weak_ptr_factory_.GetWeakPtr()));
 }
 
 void SyncSchedulerImpl::TrySyncCycleJobImpl() {
   JobPriority priority = next_sync_cycle_job_priority_;
   next_sync_cycle_job_priority_ = NORMAL_PRIORITY;
 
-  nudge_tracker_.SetSyncCycleStartTime(base::TimeTicks::Now());
+  nudge_tracker_.SetSyncCycleStartTime(TimeTicks::Now());
 
   DCHECK(CalledOnValidThread());
   if (mode_ == CONFIGURATION_MODE) {
     if (pending_configure_params_) {
       SDVLOG(2) << "Found pending configure job";
       DoConfigurationSyncCycleJob(priority);
     }
   } else if (mode_ == CLEAR_SERVER_DATA_MODE) {
     if (pending_clear_params_) {
       DoClearServerDataSyncCycleJob(priority);
     }
   } else if (CanRunNudgeJobNow(priority)) {
     if (nudge_tracker_.IsSyncRequired()) {
       SDVLOG(2) << "Found pending nudge job";
       DoNudgeSyncCycleJob(priority);
-    } else if (((base::TimeTicks::Now() - last_poll_reset_) >=
-                GetPollInterval())) {
+    } else if (((TimeTicks::Now() - last_poll_reset_) >= GetPollInterval())) {
       SDVLOG(2) << "Found pending poll";
       DoPollSyncCycleJob();
     }
   } else {
     // We must be in an error state. Transitioning out of each of these
     // error states should trigger a canary job.
     DCHECK(IsCurrentlyThrottled() || IsBackingOff() ||
            cycle_context_->connection_manager()->HasInvalidAuthToken());
   }
 
@@ skipping 21 matching lines @@
   TrySyncCycleJob();
 }
 
 void SyncSchedulerImpl::Unthrottle() {
   DCHECK(CalledOnValidThread());
   DCHECK_EQ(WaitInterval::THROTTLED, wait_interval_->mode);
 
   // We're no longer throttled, so clear the wait interval.
   wait_interval_.reset();
   NotifyRetryTime(base::Time());
-  NotifyThrottledTypesChanged(nudge_tracker_.GetThrottledTypes());
+  NotifyBlockedTypesChanged(nudge_tracker_.GetBlockedTypes());
 
   // We treat this as a 'canary' in the sense that it was originally scheduled
   // to run some time ago, failed, and we now want to retry, versus a job that
   // was just created (e.g via ScheduleNudgeImpl). The main implication is
   // that we're careful to update routing info (etc) with such potentially
   // stale canary jobs.
   TryCanaryJob();
 }
 
-void SyncSchedulerImpl::TypeUnthrottle(base::TimeTicks unthrottle_time) {
+void SyncSchedulerImpl::OnTypesUnblocked() {
   DCHECK(CalledOnValidThread());
-  nudge_tracker_.UpdateTypeThrottlingState(unthrottle_time);
-  NotifyThrottledTypesChanged(nudge_tracker_.GetThrottledTypes());
+  nudge_tracker_.UpdateTypeThrottlingAndBackoffState();
+  NotifyBlockedTypesChanged(nudge_tracker_.GetBlockedTypes());
 
-  if (nudge_tracker_.IsAnyTypeThrottled()) {
-    const base::TimeTicks now = base::TimeTicks::Now();
-    base::TimeDelta time_until_next_unthrottle =
-        nudge_tracker_.GetTimeUntilNextUnthrottle(now);
-    type_unthrottle_timer_.Start(FROM_HERE, time_until_next_unthrottle,
-                                 base::Bind(&SyncSchedulerImpl::TypeUnthrottle,
-                                            weak_ptr_factory_.GetWeakPtr(),
-                                            now + time_until_next_unthrottle));
-  }
+  RestartWaiting();
 
   // Maybe this is a good time to run a nudge job.  Let's try it.
   if (nudge_tracker_.IsSyncRequired() && CanRunNudgeJobNow(NORMAL_PRIORITY))
     TrySyncCycleJob();
 }
 
 void SyncSchedulerImpl::PerformDelayedNudge() {
   // Circumstances may have changed since we scheduled this delayed nudge.
   // We must check to see if it's OK to run the job before we do so.
   if (CanRunNudgeJobNow(NORMAL_PRIORITY))
     TrySyncCycleJob();
 
   // We're not responsible for setting up any retries here.  The functions that
   // first put us into a state that prevents successful sync cycles (eg. global
   // throttling, type throttling, network errors, transient errors) will also
   // setup the appropriate retry logic (eg. retry after timeout, exponential
   // backoff, retry when the network changes).
 }
 
 void SyncSchedulerImpl::ExponentialBackoffRetry() {
   TryCanaryJob();
 }
 
 void SyncSchedulerImpl::NotifyRetryTime(base::Time retry_time) {
   for (auto& observer : *cycle_context_->listeners())
     observer.OnRetryTimeChanged(retry_time);
 }
 
-void SyncSchedulerImpl::NotifyThrottledTypesChanged(ModelTypeSet types) {
-  for (auto& observer : *cycle_context_->listeners())
-    observer.OnThrottledTypesChanged(types);
+void SyncSchedulerImpl::NotifyBlockedTypesChanged(ModelTypeSet types) {
+  ModelTypeSet throttled_types;
+  ModelTypeSet backed_off_types;
+  for (ModelTypeSet::Iterator type_it = types.First(); type_it.Good();
+       type_it.Inc()) {
+    if (nudge_tracker_.GetTypeBlockingMode(type_it.Get()) ==
+        WaitInterval::THROTTLED) {
+      throttled_types.Put(type_it.Get());
+    } else if (nudge_tracker_.GetTypeBlockingMode(type_it.Get()) ==
+               WaitInterval::EXPONENTIAL_BACKOFF) {
+      backed_off_types.Put(type_it.Get());
+    }
+  }
+
+  for (auto& observer : *cycle_context_->listeners()) {
+    observer.OnThrottledTypesChanged(throttled_types);
+    observer.OnBackedOffTypesChanged(backed_off_types);
+  }
 }
 
 bool SyncSchedulerImpl::IsBackingOff() const {
   DCHECK(CalledOnValidThread());
   return wait_interval_.get() &&
          wait_interval_->mode == WaitInterval::EXPONENTIAL_BACKOFF;
 }
 
-void SyncSchedulerImpl::OnThrottled(const base::TimeDelta& throttle_duration) {
+void SyncSchedulerImpl::OnThrottled(const TimeDelta& throttle_duration) {
   DCHECK(CalledOnValidThread());
   wait_interval_ = base::MakeUnique<WaitInterval>(WaitInterval::THROTTLED,
                                                   throttle_duration);
   NotifyRetryTime(base::Time::Now() + wait_interval_->length);
-  NotifyThrottledTypesChanged(ModelTypeSet::All());
+
+  for (auto& observer : *cycle_context_->listeners()) {
+    observer.OnThrottledTypesChanged(ModelTypeSet::All());
+  }
 }
 
-void SyncSchedulerImpl::OnTypesThrottled(
-    ModelTypeSet types,
-    const base::TimeDelta& throttle_duration) {
-  base::TimeTicks now = base::TimeTicks::Now();
+void SyncSchedulerImpl::OnTypesThrottled(ModelTypeSet types,
+                                         const TimeDelta& throttle_duration) {
+  TimeTicks now = TimeTicks::Now();
 
   SDVLOG(1) << "Throttling " << ModelTypeSetToString(types) << " for "
             << throttle_duration.InMinutes() << " minutes.";
 
   nudge_tracker_.SetTypesThrottledUntil(types, throttle_duration, now);
-  base::TimeDelta time_until_next_unthrottle =
-      nudge_tracker_.GetTimeUntilNextUnthrottle(now);
-  type_unthrottle_timer_.Start(FROM_HERE, time_until_next_unthrottle,
-                               base::Bind(&SyncSchedulerImpl::TypeUnthrottle,
-                                          weak_ptr_factory_.GetWeakPtr(),
-                                          now + time_until_next_unthrottle));
-  NotifyThrottledTypesChanged(nudge_tracker_.GetThrottledTypes());
+  RestartWaiting();
+  NotifyBlockedTypesChanged(nudge_tracker_.GetBlockedTypes());
+}
+
+void SyncSchedulerImpl::OnTypesBackedOff(ModelTypeSet types) {
+  TimeTicks now = TimeTicks::Now();
+
+  for (ModelTypeSet::Iterator type = types.First(); type.Good(); type.Inc()) {
+    TimeDelta last_backoff_time =
+        TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
+    if (nudge_tracker_.GetTypeBlockingMode(type.Get()) ==
+        WaitInterval::EXPONENTIAL_BACKOFF_RETRYING) {
+      last_backoff_time = nudge_tracker_.GetTypeLastBackoffInterval(type.Get());
+    }
+
+    TimeDelta length = delay_provider_->GetDelay(last_backoff_time);
+    nudge_tracker_.SetTypeBackedOff(type.Get(), length, now);
+    SDVLOG(1) << "Backing off " << ModelTypeToString(type.Get()) << " for "
+              << length.InSeconds() << " second.";
+  }
+  RestartWaiting();
+  NotifyBlockedTypesChanged(nudge_tracker_.GetBlockedTypes());
 }
 
 bool SyncSchedulerImpl::IsCurrentlyThrottled() {
   DCHECK(CalledOnValidThread());
   return wait_interval_.get() &&
          wait_interval_->mode == WaitInterval::THROTTLED;
 }
 
 void SyncSchedulerImpl::OnReceivedShortPollIntervalUpdate(
-    const base::TimeDelta& new_interval) {
+    const TimeDelta& new_interval) {
   DCHECK(CalledOnValidThread());
   if (new_interval == syncer_short_poll_interval_seconds_)
     return;
   SDVLOG(1) << "Updating short poll interval to " << new_interval.InMinutes()
             << " minutes.";
   syncer_short_poll_interval_seconds_ = new_interval;
   AdjustPolling(UPDATE_INTERVAL);
 }
 
 void SyncSchedulerImpl::OnReceivedLongPollIntervalUpdate(
-    const base::TimeDelta& new_interval) {
+    const TimeDelta& new_interval) {
   DCHECK(CalledOnValidThread());
   if (new_interval == syncer_long_poll_interval_seconds_)
     return;
   SDVLOG(1) << "Updating long poll interval to " << new_interval.InMinutes()
             << " minutes.";
   syncer_long_poll_interval_seconds_ = new_interval;
   AdjustPolling(UPDATE_INTERVAL);
 }
 
 void SyncSchedulerImpl::OnReceivedCustomNudgeDelays(
-    const std::map<ModelType, base::TimeDelta>& nudge_delays) {
+    const std::map<ModelType, TimeDelta>& nudge_delays) {
   DCHECK(CalledOnValidThread());
   nudge_tracker_.OnReceivedCustomNudgeDelays(nudge_delays);
 }
 
 void SyncSchedulerImpl::OnReceivedClientInvalidationHintBufferSize(int size) {
   if (size > 0)
     nudge_tracker_.SetHintBufferSize(size);
   else
     NOTREACHED() << "Hint buffer size should be > 0.";
 }
 
 void SyncSchedulerImpl::OnSyncProtocolError(
     const SyncProtocolError& sync_protocol_error) {
   DCHECK(CalledOnValidThread());
   if (ShouldRequestEarlyExit(sync_protocol_error)) {
     SDVLOG(2) << "Sync Scheduler requesting early exit.";
     Stop();
   }
   if (IsActionableError(sync_protocol_error)) {
     SDVLOG(2) << "OnActionableError";
     for (auto& observer : *cycle_context_->listeners())
       observer.OnActionableError(sync_protocol_error);
   }
 }
 
-void SyncSchedulerImpl::OnReceivedGuRetryDelay(const base::TimeDelta& delay) {
+void SyncSchedulerImpl::OnReceivedGuRetryDelay(const TimeDelta& delay) {
   nudge_tracker_.SetNextRetryTime(TimeTicks::Now() + delay);
   retry_timer_.Start(FROM_HERE, delay, this,
                      &SyncSchedulerImpl::RetryTimerCallback);
 }
 
 void SyncSchedulerImpl::OnReceivedMigrationRequest(ModelTypeSet types) {
   for (auto& observer : *cycle_context_->listeners())
     observer.OnMigrationRequested(types);
 }
 
 void SyncSchedulerImpl::SetNotificationsEnabled(bool notifications_enabled) {
   DCHECK(CalledOnValidThread());
   cycle_context_->set_notifications_enabled(notifications_enabled);
   if (notifications_enabled)
     nudge_tracker_.OnInvalidationsEnabled();
   else
     nudge_tracker_.OnInvalidationsDisabled();
 }
 
 #undef SDVLOG_LOC
 
 #undef SDVLOG
 
 #undef SLOG
 
 #undef ENUM_CASE
 
 }  // namespace syncer