[Sync] Sync client should to exponential backoff when receive partial failure

components/sync/engine_impl/sync_scheduler_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/sync/engine_impl/sync_scheduler_impl.h"
 
 #include <algorithm>
 #include <cstring>
 #include <utility>
 
@@ skipping 70 matching lines @@
 }
 
 void RunAndReset(base::Closure* task) {
   DCHECK(task);
   if (task->is_null())
     return;
   task->Run();
   task->Reset();
 }
 
+#define ENUM_CASE(x) \
+  case x:            \
+    return #x;       \
+    break;
+
 }  // namespace
 
 ConfigurationParams::ConfigurationParams()
     : source(GetUpdatesCallerInfo::UNKNOWN) {}
 ConfigurationParams::ConfigurationParams(
     const sync_pb::GetUpdatesCallerInfo::GetUpdatesSource& source,
     ModelTypeSet types_to_download,
     const ModelSafeRoutingInfo& routing_info,
     const base::Closure& ready_task,
     const base::Closure& retry_task)
     : source(source),
       types_to_download(types_to_download),
       routing_info(routing_info),
       ready_task(ready_task),
       retry_task(retry_task) {
   DCHECK(!ready_task.is_null());
 }
 ConfigurationParams::ConfigurationParams(const ConfigurationParams& other) =
     default;
 ConfigurationParams::~ConfigurationParams() {}
 
 ClearParams::ClearParams(const base::Closure& report_success_task)
     : report_success_task(report_success_task) {
   DCHECK(!report_success_task.is_null());
 }
 ClearParams::ClearParams(const ClearParams& other) = default;
 ClearParams::~ClearParams() {}
 
-SyncSchedulerImpl::WaitInterval::WaitInterval() : mode(UNKNOWN) {}
-
-SyncSchedulerImpl::WaitInterval::WaitInterval(Mode mode, TimeDelta length)
-    : mode(mode), length(length) {}
-
-SyncSchedulerImpl::WaitInterval::~WaitInterval() {}
-
-#define ENUM_CASE(x) \
-  case x:            \
-    return #x;       \
-    break;
-
-const char* SyncSchedulerImpl::WaitInterval::GetModeString(Mode mode) {
-  switch (mode) {
-    ENUM_CASE(UNKNOWN);
-    ENUM_CASE(EXPONENTIAL_BACKOFF);
-    ENUM_CASE(THROTTLED);
-  }
-  NOTREACHED();
-  return "";
-}
-
 GetUpdatesCallerInfo::GetUpdatesSource GetUpdatesFromNudgeSource(
     NudgeSource source) {
   switch (source) {
     case NUDGE_SOURCE_NOTIFICATION:
       return GetUpdatesCallerInfo::NOTIFICATION;
     case NUDGE_SOURCE_LOCAL:
       return GetUpdatesCallerInfo::LOCAL;
     case NUDGE_SOURCE_LOCAL_REFRESH:
       return GetUpdatesCallerInfo::DATATYPE_REFRESH;
     case NUDGE_SOURCE_UNKNOWN:
@@ skipping 93 matching lines @@
   Mode old_mode = mode_;
   mode_ = mode;
   // Only adjust the poll reset time if it was valid and in the past.
   if (!last_poll_time.is_null() && last_poll_time < base::Time::Now()) {
     // Convert from base::Time to base::TimeTicks. The reason we use Time
     // for persisting is that TimeTicks can stop making forward progress when
     // the machine is suspended. This implies that on resume the client might
     // actually have miss the real poll, unless the client is restarted. Fixing
     // that would require using an AlarmTimer though, which is only supported
     // on certain platforms.
-    last_poll_reset_ =
-        base::TimeTicks::Now() - (base::Time::Now() - last_poll_time);
+    last_poll_reset_ = TimeTicks::Now() - (base::Time::Now() - last_poll_time);
   }
 
   if (old_mode != mode_ && mode_ == NORMAL_MODE) {
     // We just got back to normal mode.  Let's try to run the work that was
     // queued up while we were configuring.
 
     AdjustPolling(UPDATE_INTERVAL);  // Will kick start poll timer if needed.
 
     // Update our current time before checking IsRetryRequired().
-    nudge_tracker_.SetSyncCycleStartTime(base::TimeTicks::Now());
+    nudge_tracker_.SetSyncCycleStartTime(TimeTicks::Now());
     if (nudge_tracker_.IsSyncRequired() && CanRunNudgeJobNow(NORMAL_PRIORITY)) {
       TrySyncCycleJob();
     }
   }
 }
 
-ModelTypeSet SyncSchedulerImpl::GetEnabledAndUnthrottledTypes() {
+ModelTypeSet SyncSchedulerImpl::GetEnabledAndUnblockedTypes() {
   ModelTypeSet enabled_types = cycle_context_->GetEnabledTypes();
   ModelTypeSet enabled_protocol_types =
       Intersection(ProtocolTypes(), enabled_types);
   ModelTypeSet throttled_types = nudge_tracker_.GetThrottledTypes();
-  return Difference(enabled_protocol_types, throttled_types);
+  ModelTypeSet backed_off_types = nudge_tracker_.GetBackedOffTypes();
+  return Difference(enabled_protocol_types,
+                    Union(throttled_types, backed_off_types));
 }
 
 void SyncSchedulerImpl::SendInitialSnapshot() {
   DCHECK(CalledOnValidThread());
   std::unique_ptr<SyncCycle> dummy(SyncCycle::Build(cycle_context_, this));
   SyncCycleEvent event(SyncCycleEvent::STATUS_CHANGED);
   event.snapshot = dummy->TakeSnapshot();
   for (auto& observer : *cycle_context_->listeners())
     observer.OnSyncCycleEvent(event);
 }
@@ skipping 99 matching lines @@
 
 void SyncSchedulerImpl::ScheduleLocalNudge(
     ModelTypeSet types,
     const tracked_objects::Location& nudge_location) {
   DCHECK(CalledOnValidThread());
   DCHECK(!types.Empty());
 
   SDVLOG_LOC(nudge_location, 2) << "Scheduling sync because of local change to "
                                 << ModelTypeSetToString(types);
   UpdateNudgeTimeRecords(types);
-  base::TimeDelta nudge_delay = nudge_tracker_.RecordLocalChange(types);
+  TimeDelta nudge_delay = nudge_tracker_.RecordLocalChange(types);
   ScheduleNudgeImpl(nudge_delay, nudge_location);
 }
 
 void SyncSchedulerImpl::ScheduleLocalRefreshRequest(
     ModelTypeSet types,
     const tracked_objects::Location& nudge_location) {
   DCHECK(CalledOnValidThread());
   DCHECK(!types.Empty());
 
   SDVLOG_LOC(nudge_location, 2)
       << "Scheduling sync because of local refresh request for "
       << ModelTypeSetToString(types);
-  base::TimeDelta nudge_delay = nudge_tracker_.RecordLocalRefreshRequest(types);
+  TimeDelta nudge_delay = nudge_tracker_.RecordLocalRefreshRequest(types);
   ScheduleNudgeImpl(nudge_delay, nudge_location);
 }
 
 void SyncSchedulerImpl::ScheduleInvalidationNudge(
     ModelType model_type,
     std::unique_ptr<InvalidationInterface> invalidation,
     const tracked_objects::Location& nudge_location) {
   DCHECK(CalledOnValidThread());
 
   SDVLOG_LOC(nudge_location, 2)
       << "Scheduling sync because we received invalidation for "
       << ModelTypeToString(model_type);
-  base::TimeDelta nudge_delay = nudge_tracker_.RecordRemoteInvalidation(
+  TimeDelta nudge_delay = nudge_tracker_.RecordRemoteInvalidation(
       model_type, std::move(invalidation));
   ScheduleNudgeImpl(nudge_delay, nudge_location);
 }
 
 void SyncSchedulerImpl::ScheduleInitialSyncNudge(ModelType model_type) {
   DCHECK(CalledOnValidThread());
 
   SDVLOG(2) << "Scheduling non-blocking initial sync for "
             << ModelTypeToString(model_type);
   nudge_tracker_.RecordInitialSyncRequired(model_type);
@@ skipping 14 matching lines @@
     return;
   }
 
   SDVLOG_LOC(nudge_location, 2) << "In ScheduleNudgeImpl with delay "
                                 << delay.InMilliseconds() << " ms";
 
   if (!CanRunNudgeJobNow(NORMAL_PRIORITY))
     return;
 
   TimeTicks incoming_run_time = TimeTicks::Now() + delay;
-  if (!scheduled_nudge_time_.is_null() &&
-      (scheduled_nudge_time_ < incoming_run_time)) {
+  if (global_wakeup_timer_.IsRunning() &&
+      (global_wakeup_timer_.desired_run_time() < incoming_run_time)) {
     // Old job arrives sooner than this one.  Don't reschedule it.
     return;
   }
 
   // Either there is no existing nudge in flight or the incoming nudge should be
   // made to arrive first (preempt) the existing nudge.  We reschedule in either
   // case.
   SDVLOG_LOC(nudge_location, 2) << "Scheduling a nudge with "
                                 << delay.InMilliseconds() << " ms delay";
-  scheduled_nudge_time_ = incoming_run_time;
-  pending_wakeup_timer_.Start(
-      nudge_location, delay, base::Bind(&SyncSchedulerImpl::PerformDelayedNudge,
+  global_wakeup_timer_.Start(nudge_location, delay,
+                             base::Bind(&SyncSchedulerImpl::PerformDelayedNudge,
                                         weak_ptr_factory_.GetWeakPtr()));
 }
 
 const char* SyncSchedulerImpl::GetModeString(SyncScheduler::Mode mode) {
   switch (mode) {
     ENUM_CASE(CONFIGURATION_MODE);
     ENUM_CASE(CLEAR_SERVER_DATA_MODE);
     ENUM_CASE(NORMAL_MODE);
   }
   return "";
 }
 
-void SyncSchedulerImpl::SetDefaultNudgeDelay(base::TimeDelta delay_ms) {
+void SyncSchedulerImpl::SetDefaultNudgeDelay(TimeDelta delay_ms) {
   DCHECK(CalledOnValidThread());
   nudge_tracker_.SetDefaultNudgeDelay(delay_ms);
 }
 
 void SyncSchedulerImpl::DoNudgeSyncCycleJob(JobPriority priority) {
   DCHECK(CalledOnValidThread());
   DCHECK(CanRunNudgeJobNow(priority));
 
   DVLOG(2) << "Will run normal mode sync cycle with types "
            << ModelTypeSetToString(cycle_context_->GetEnabledTypes());
   std::unique_ptr<SyncCycle> cycle(SyncCycle::Build(cycle_context_, this));
-  bool success = syncer_->NormalSyncShare(GetEnabledAndUnthrottledTypes(),
+  bool success = syncer_->NormalSyncShare(GetEnabledAndUnblockedTypes(),
                                           &nudge_tracker_, cycle.get());
 
   if (success) {
     // That cycle took care of any outstanding work we had.
     SDVLOG(2) << "Nudge succeeded.";
     nudge_tracker_.RecordSuccessfulSyncCycle();
-    scheduled_nudge_time_ = base::TimeTicks();
     HandleSuccess();
 
     // If this was a canary, we may need to restart the poll timer (the poll
     // timer may have fired while the scheduler was in an error state, ignoring
     // the poll).
     if (!poll_timer_.IsRunning()) {
       SDVLOG(1) << "Canary succeeded, restarting polling.";
       AdjustPolling(UPDATE_INTERVAL);
     }
   } else {
@@ skipping 57 matching lines @@
 }
 
 void SyncSchedulerImpl::HandleSuccess() {
   // If we're here, then we successfully reached the server.  End all backoff.
   wait_interval_.reset();
   NotifyRetryTime(base::Time());
 }
 
 void SyncSchedulerImpl::HandleFailure(
     const ModelNeutralState& model_neutral_state) {
+  if (model_neutral_state.commit_result == SERVER_RETURN_PARTIAL_FAILURE) {

[Nicolas Zea, 2016/11/09 00:21:25]

Should partial failure just count as success?

[Gang Wu, 2016/11/10 21:56:51]

Done.

+    // Since it is partial failure, we have handled in OnTypesThrottled or
+    // OnTypesBackedOff already.
+    return;
+  }
   if (IsCurrentlyThrottled()) {
     SDVLOG(2) << "Was throttled during previous sync cycle.";
   } else if (!IsBackingOff()) {
     // Setup our backoff if this is our first such failure.
     TimeDelta length = delay_provider_->GetDelay(
         delay_provider_->GetInitialDelay(model_neutral_state));
     wait_interval_ = base::MakeUnique<WaitInterval>(
         WaitInterval::EXPONENTIAL_BACKOFF, length);
     SDVLOG(2) << "Sync cycle failed.  Will back off for "
               << wait_interval_->length.InMilliseconds() << "ms.";
   } else {
     // Increase our backoff interval and schedule another retry.
     TimeDelta length = delay_provider_->GetDelay(wait_interval_->length);
     wait_interval_ = base::MakeUnique<WaitInterval>(
         WaitInterval::EXPONENTIAL_BACKOFF, length);
     SDVLOG(2) << "Sync cycle failed.  Will back off for "
               << wait_interval_->length.InMilliseconds() << "ms.";
   }
   RestartWaiting();
 }
 
 void SyncSchedulerImpl::DoPollSyncCycleJob() {
   SDVLOG(2) << "Polling with types "
-            << ModelTypeSetToString(GetEnabledAndUnthrottledTypes());
+            << ModelTypeSetToString(GetEnabledAndUnblockedTypes());
   std::unique_ptr<SyncCycle> cycle(SyncCycle::Build(cycle_context_, this));
   bool success =
-      syncer_->PollSyncShare(GetEnabledAndUnthrottledTypes(), cycle.get());
+      syncer_->PollSyncShare(GetEnabledAndUnblockedTypes(), cycle.get());
 
   // Only restart the timer if the poll succeeded. Otherwise rely on normal
   // failure handling to retry with backoff.
   if (success) {
     AdjustPolling(FORCE_RESET);
     HandleSuccess();
   } else {
     HandleFailure(cycle->status_controller().model_neutral_state());
   }
 }
 
 void SyncSchedulerImpl::UpdateNudgeTimeRecords(ModelTypeSet types) {
   DCHECK(CalledOnValidThread());
-  base::TimeTicks now = TimeTicks::Now();
+  TimeTicks now = TimeTicks::Now();
   // Update timing information for how often datatypes are triggering nudges.
   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
-    base::TimeTicks previous = last_local_nudges_by_model_type_[iter.Get()];
+    TimeTicks previous = last_local_nudges_by_model_type_[iter.Get()];
     last_local_nudges_by_model_type_[iter.Get()] = now;
     if (previous.is_null())
       continue;
 
 #define PER_DATA_TYPE_MACRO(type_str) \
   SYNC_FREQ_HISTOGRAM("Sync.Freq" type_str, now - previous);
     SYNC_DATA_TYPE_HISTOGRAM(iter.Get());
 #undef PER_DATA_TYPE_MACRO
   }
 }
@@ skipping 48 matching lines @@
                     &SyncSchedulerImpl::PollTimerCallback);
 }
 
 void SyncSchedulerImpl::RestartWaiting() {
   CHECK(wait_interval_.get());
   DCHECK(wait_interval_->length >= TimeDelta::FromSeconds(0));
   NotifyRetryTime(base::Time::Now() + wait_interval_->length);
   SDVLOG(2) << "Starting WaitInterval timer of length "
             << wait_interval_->length.InMilliseconds() << "ms.";
   if (wait_interval_->mode == WaitInterval::THROTTLED) {
-    pending_wakeup_timer_.Start(FROM_HERE, wait_interval_->length,
-                                base::Bind(&SyncSchedulerImpl::Unthrottle,
-                                           weak_ptr_factory_.GetWeakPtr()));
+    global_wakeup_timer_.Start(FROM_HERE, wait_interval_->length,
+                               base::Bind(&SyncSchedulerImpl::Unthrottle,
+                                          weak_ptr_factory_.GetWeakPtr()));
   } else {
-    pending_wakeup_timer_.Start(
+    global_wakeup_timer_.Start(
         FROM_HERE, wait_interval_->length,
         base::Bind(&SyncSchedulerImpl::ExponentialBackoffRetry,
                    weak_ptr_factory_.GetWeakPtr()));
   }
 }
 
 void SyncSchedulerImpl::Stop() {
   DCHECK(CalledOnValidThread());
   SDVLOG(2) << "Stop called";
 
   // Kill any in-flight method calls.
   weak_ptr_factory_.InvalidateWeakPtrs();
   wait_interval_.reset();
   NotifyRetryTime(base::Time());
   poll_timer_.Stop();
-  pending_wakeup_timer_.Stop();
+  global_wakeup_timer_.Stop();
+  type_wakeup_timer_.Stop();
   pending_configure_params_.reset();
   pending_clear_params_.reset();
   if (started_)
     started_ = false;
 }
 
 // This is the only place where we invoke DoSyncCycleJob with canary
 // privileges.  Everyone else should use NORMAL_PRIORITY.
 void SyncSchedulerImpl::TryCanaryJob() {
   next_sync_cycle_job_priority_ = CANARY_PRIORITY;
   SDVLOG(2) << "Attempting canary job";
   TrySyncCycleJob();
 }
 
 void SyncSchedulerImpl::TrySyncCycleJob() {
   // Post call to TrySyncCycleJobImpl on current thread. Later request for
   // access token will be here.
   base::ThreadTaskRunnerHandle::Get()->PostTask(
       FROM_HERE, base::Bind(&SyncSchedulerImpl::TrySyncCycleJobImpl,
                             weak_ptr_factory_.GetWeakPtr()));
 }
 
 void SyncSchedulerImpl::TrySyncCycleJobImpl() {
   JobPriority priority = next_sync_cycle_job_priority_;
   next_sync_cycle_job_priority_ = NORMAL_PRIORITY;
 
-  nudge_tracker_.SetSyncCycleStartTime(base::TimeTicks::Now());
+  nudge_tracker_.SetSyncCycleStartTime(TimeTicks::Now());
 
   DCHECK(CalledOnValidThread());
   if (mode_ == CONFIGURATION_MODE) {
     if (pending_configure_params_) {
       SDVLOG(2) << "Found pending configure job";
       DoConfigurationSyncCycleJob(priority);
     }
   } else if (mode_ == CLEAR_SERVER_DATA_MODE) {
     if (pending_clear_params_) {
       DoClearServerDataSyncCycleJob(priority);
     }
   } else if (CanRunNudgeJobNow(priority)) {
     if (nudge_tracker_.IsSyncRequired()) {
       SDVLOG(2) << "Found pending nudge job";
       DoNudgeSyncCycleJob(priority);
-    } else if (((base::TimeTicks::Now() - last_poll_reset_) >=
-                GetPollInterval())) {
+    } else if (((TimeTicks::Now() - last_poll_reset_) >= GetPollInterval())) {
       SDVLOG(2) << "Found pending poll";
       DoPollSyncCycleJob();
     }
   } else {
     // We must be in an error state. Transitioning out of each of these
     // error states should trigger a canary job.
     DCHECK(IsCurrentlyThrottled() || IsBackingOff() ||
            cycle_context_->connection_manager()->HasInvalidAuthToken());
   }
 
-  if (IsBackingOff() && !pending_wakeup_timer_.IsRunning()) {
+  if (IsBackingOff() && !global_wakeup_timer_.IsRunning()) {
     // If we succeeded, our wait interval would have been cleared.  If it hasn't
     // been cleared, then we should increase our backoff interval and schedule
     // another retry.
     TimeDelta length = delay_provider_->GetDelay(wait_interval_->length);
     wait_interval_ = base::MakeUnique<WaitInterval>(
         WaitInterval::EXPONENTIAL_BACKOFF, length);
     SDVLOG(2) << "Sync cycle failed.  Will back off for "
               << wait_interval_->length.InMilliseconds() << "ms.";
     RestartWaiting();
   }
@@ skipping 20 matching lines @@
   NotifyThrottledTypesChanged(nudge_tracker_.GetThrottledTypes());
 
   // We treat this as a 'canary' in the sense that it was originally scheduled
   // to run some time ago, failed, and we now want to retry, versus a job that
   // was just created (e.g via ScheduleNudgeImpl). The main implication is
   // that we're careful to update routing info (etc) with such potentially
   // stale canary jobs.
   TryCanaryJob();
 }
 
-void SyncSchedulerImpl::TypeUnthrottle(base::TimeTicks unthrottle_time) {
+void SyncSchedulerImpl::TypesUnblock(TimeTicks unbackoff_time) {

[Nicolas Zea, 2016/11/09 00:21:25]

I think naming this method something like OnTypesUnblocked would be clearer.

[Gang Wu, 2016/11/10 21:56:51]

Done.

   DCHECK(CalledOnValidThread());
-  nudge_tracker_.UpdateTypeThrottlingState(unthrottle_time);
+  nudge_tracker_.UpdateTypeThrottlingAndBackoffState(unbackoff_time);
   NotifyThrottledTypesChanged(nudge_tracker_.GetThrottledTypes());
+  NotifyBackedOffTypesChanged(nudge_tracker_.GetBackedOffTypes());
 
+  // If there are some datatype still throttling or backoff, find the closest

[Nicolas Zea, 2016/11/09 00:21:25]

"still throttling or backoff" -> "still throttled or backed off"

[Gang Wu, 2016/11/10 21:56:51]

Done.

+  // time and schedule SyncSchedulerImpl::TypesUnblock with the closest time.
+  const TimeTicks now = TimeTicks::Now();
+  TimeDelta time_until_next_unblock = TimeDelta::Max();
   if (nudge_tracker_.IsAnyTypeThrottled()) {
-    const base::TimeTicks now = base::TimeTicks::Now();
-    base::TimeDelta time_until_next_unthrottle =
-        nudge_tracker_.GetTimeUntilNextUnthrottle(now);
-    type_unthrottle_timer_.Start(FROM_HERE, time_until_next_unthrottle,
-                                 base::Bind(&SyncSchedulerImpl::TypeUnthrottle,
-                                            weak_ptr_factory_.GetWeakPtr(),
-                                            now + time_until_next_unthrottle));
+    time_until_next_unblock = nudge_tracker_.GetTimeUntilNextUnthrottle(now);
+  }
+  if (nudge_tracker_.IsAnyTypeBackedOff()) {
+    time_until_next_unblock = std::min(
+        time_until_next_unblock, nudge_tracker_.GetTimeUntilNextUnbackoff(now));
+  }
+  if (!time_until_next_unblock.is_max()) {
+    type_wakeup_timer_.Start(FROM_HERE, time_until_next_unblock,
+                             base::Bind(&SyncSchedulerImpl::TypesUnblock,
+                                        weak_ptr_factory_.GetWeakPtr(),
+                                        now + time_until_next_unblock));

[Nicolas Zea, 2016/11/09 00:21:25]

as mentioned earlier, it's not clear to me why we need to pass an argument to
TypesUnlock. Given the timer will call it at the right time, we should I think
be able to rely on base::TimeTicks::Now()

[Gang Wu, 2016/11/10 21:56:51]

Done.

   }
 
   // Maybe this is a good time to run a nudge job.  Let's try it.
   if (nudge_tracker_.IsSyncRequired() && CanRunNudgeJobNow(NORMAL_PRIORITY))
     TrySyncCycleJob();
 }
 
 void SyncSchedulerImpl::PerformDelayedNudge() {
   // Circumstances may have changed since we scheduled this delayed nudge.
   // We must check to see if it's OK to run the job before we do so.
@@ skipping 14 matching lines @@
 void SyncSchedulerImpl::NotifyRetryTime(base::Time retry_time) {
   for (auto& observer : *cycle_context_->listeners())
     observer.OnRetryTimeChanged(retry_time);
 }
 
 void SyncSchedulerImpl::NotifyThrottledTypesChanged(ModelTypeSet types) {
   for (auto& observer : *cycle_context_->listeners())
     observer.OnThrottledTypesChanged(types);
 }
 
+void SyncSchedulerImpl::NotifyBackedOffTypesChanged(ModelTypeSet types) {
+  for (auto& observer : *cycle_context_->listeners())
+    observer.OnBackedOffTypesChanged(types);
+}
+
 bool SyncSchedulerImpl::IsBackingOff() const {
   DCHECK(CalledOnValidThread());
   return wait_interval_.get() &&
          wait_interval_->mode == WaitInterval::EXPONENTIAL_BACKOFF;
 }
 
-void SyncSchedulerImpl::OnThrottled(const base::TimeDelta& throttle_duration) {
+void SyncSchedulerImpl::OnThrottled(const TimeDelta& throttle_duration) {
   DCHECK(CalledOnValidThread());
   wait_interval_ = base::MakeUnique<WaitInterval>(WaitInterval::THROTTLED,
                                                   throttle_duration);
   NotifyRetryTime(base::Time::Now() + wait_interval_->length);
   NotifyThrottledTypesChanged(ModelTypeSet::All());
 }
 
-void SyncSchedulerImpl::OnTypesThrottled(
-    ModelTypeSet types,
-    const base::TimeDelta& throttle_duration) {
-  base::TimeTicks now = base::TimeTicks::Now();
+void SyncSchedulerImpl::OnTypesThrottled(ModelTypeSet types,
+                                         const TimeDelta& throttle_duration) {
+  TimeTicks now = TimeTicks::Now();
 
   SDVLOG(1) << "Throttling " << ModelTypeSetToString(types) << " for "
             << throttle_duration.InMinutes() << " minutes.";
 
   nudge_tracker_.SetTypesThrottledUntil(types, throttle_duration, now);
-  base::TimeDelta time_until_next_unthrottle =
+  TimeDelta time_until_next_unthrottle =
       nudge_tracker_.GetTimeUntilNextUnthrottle(now);
-  type_unthrottle_timer_.Start(FROM_HERE, time_until_next_unthrottle,
-                               base::Bind(&SyncSchedulerImpl::TypeUnthrottle,
-                                          weak_ptr_factory_.GetWeakPtr(),
-                                          now + time_until_next_unthrottle));
+  type_wakeup_timer_.Start(FROM_HERE, time_until_next_unthrottle,

[Nicolas Zea, 2016/11/09 00:21:25]

Why do we need a separate timer? Can't we reuse the global timer?

[Gang Wu, 2016/11/10 21:56:51]

Done.

+                           base::Bind(&SyncSchedulerImpl::TypesUnblock,
+                                      weak_ptr_factory_.GetWeakPtr(),
+                                      now + time_until_next_unthrottle));
   NotifyThrottledTypesChanged(nudge_tracker_.GetThrottledTypes());
 }
 
+void SyncSchedulerImpl::OnTypesBackedOff(ModelTypeSet types) {
+  TimeTicks now = TimeTicks::Now();
+
+  for (ModelTypeSet::Iterator type = types.First(); type.Good(); type.Inc()) {
+    TimeDelta last_backoff_time =
+        TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
+    if (nudge_tracker_.IsTypeBackedOff(type.Get())) {
+      last_backoff_time = nudge_tracker_.GetTypeLastBackoffInterval(type.Get());
+    }
+
+    TimeDelta length = delay_provider_->GetDelay(last_backoff_time);
+    nudge_tracker_.SetTypeBackedOff(type.Get(), length, now);
+    SDVLOG(1) << "Backing off " << ModelTypeToString(type.Get()) << " for "
+              << length.InSeconds() << " second.";
+  }
+  TimeDelta time_until_next_unbackoff =
+      nudge_tracker_.GetTimeUntilNextUnbackoff(now);
+  type_wakeup_timer_.Start(FROM_HERE, time_until_next_unbackoff,
+                           base::Bind(&SyncSchedulerImpl::TypesUnblock,
+                                      weak_ptr_factory_.GetWeakPtr(),
+                                      now + time_until_next_unbackoff));
+  NotifyBackedOffTypesChanged(nudge_tracker_.GetBackedOffTypes());
+}
+
 bool SyncSchedulerImpl::IsCurrentlyThrottled() {
   DCHECK(CalledOnValidThread());
   return wait_interval_.get() &&
          wait_interval_->mode == WaitInterval::THROTTLED;
 }
 
 void SyncSchedulerImpl::OnReceivedShortPollIntervalUpdate(
-    const base::TimeDelta& new_interval) {
+    const TimeDelta& new_interval) {
   DCHECK(CalledOnValidThread());
   if (new_interval == syncer_short_poll_interval_seconds_)
     return;
   SDVLOG(1) << "Updating short poll interval to " << new_interval.InMinutes()
             << " minutes.";
   syncer_short_poll_interval_seconds_ = new_interval;
   AdjustPolling(UPDATE_INTERVAL);
 }
 
 void SyncSchedulerImpl::OnReceivedLongPollIntervalUpdate(
-    const base::TimeDelta& new_interval) {
+    const TimeDelta& new_interval) {
   DCHECK(CalledOnValidThread());
   if (new_interval == syncer_long_poll_interval_seconds_)
     return;
   SDVLOG(1) << "Updating long poll interval to " << new_interval.InMinutes()
             << " minutes.";
   syncer_long_poll_interval_seconds_ = new_interval;
   AdjustPolling(UPDATE_INTERVAL);
 }
 
 void SyncSchedulerImpl::OnReceivedCustomNudgeDelays(
-    const std::map<ModelType, base::TimeDelta>& nudge_delays) {
+    const std::map<ModelType, TimeDelta>& nudge_delays) {
   DCHECK(CalledOnValidThread());
   nudge_tracker_.OnReceivedCustomNudgeDelays(nudge_delays);
 }
 
 void SyncSchedulerImpl::OnReceivedClientInvalidationHintBufferSize(int size) {
   if (size > 0)
     nudge_tracker_.SetHintBufferSize(size);
   else
     NOTREACHED() << "Hint buffer size should be > 0.";
 }
 
 void SyncSchedulerImpl::OnSyncProtocolError(
     const SyncProtocolError& sync_protocol_error) {
   DCHECK(CalledOnValidThread());
   if (ShouldRequestEarlyExit(sync_protocol_error)) {
     SDVLOG(2) << "Sync Scheduler requesting early exit.";
     Stop();
   }
   if (IsActionableError(sync_protocol_error)) {
     SDVLOG(2) << "OnActionableError";
     for (auto& observer : *cycle_context_->listeners())
       observer.OnActionableError(sync_protocol_error);
   }
 }
 
-void SyncSchedulerImpl::OnReceivedGuRetryDelay(const base::TimeDelta& delay) {
+void SyncSchedulerImpl::OnReceivedGuRetryDelay(const TimeDelta& delay) {
   nudge_tracker_.SetNextRetryTime(TimeTicks::Now() + delay);
   retry_timer_.Start(FROM_HERE, delay, this,
                      &SyncSchedulerImpl::RetryTimerCallback);
 }
 
 void SyncSchedulerImpl::OnReceivedMigrationRequest(ModelTypeSet types) {
   for (auto& observer : *cycle_context_->listeners())
     observer.OnMigrationRequested(types);
 }
 
 void SyncSchedulerImpl::SetNotificationsEnabled(bool notifications_enabled) {
   DCHECK(CalledOnValidThread());
   cycle_context_->set_notifications_enabled(notifications_enabled);
   if (notifications_enabled)
     nudge_tracker_.OnInvalidationsEnabled();
   else
     nudge_tracker_.OnInvalidationsDisabled();
 }
 
 #undef SDVLOG_LOC
 
 #undef SDVLOG
 
 #undef SLOG
 
 #undef ENUM_CASE
 
 }  // namespace syncer