[Sync] Sync client should to exponential backoff when receive partial failure

components/sync/engine_impl/sync_scheduler_impl.h

 // Copyright 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef COMPONENTS_SYNC_ENGINE_IMPL_SYNC_SCHEDULER_IMPL_H_
 #define COMPONENTS_SYNC_ENGINE_IMPL_SYNC_SCHEDULER_IMPL_H_
 
 #include <map>
 #include <memory>
 #include <string>
@@ skipping 52 matching lines @@
   void ScheduleInitialSyncNudge(ModelType model_type) override;
   void SetNotificationsEnabled(bool notifications_enabled) override;
 
   void OnCredentialsUpdated() override;
   void OnConnectionStatusChange() override;
 
   // SyncCycle::Delegate implementation.
   void OnThrottled(const base::TimeDelta& throttle_duration) override;
   void OnTypesThrottled(ModelTypeSet types,
                         const base::TimeDelta& throttle_duration) override;
+  void OnTypesBackedOff(ModelTypeSet types) override;
   bool IsCurrentlyThrottled() override;
   void OnReceivedShortPollIntervalUpdate(
       const base::TimeDelta& new_interval) override;
   void OnReceivedLongPollIntervalUpdate(
       const base::TimeDelta& new_interval) override;
   void OnReceivedCustomNudgeDelays(
       const std::map<ModelType, base::TimeDelta>& nudge_delays) override;
   void OnReceivedClientInvalidationHintBufferSize(int size) override;
   void OnSyncProtocolError(
       const SyncProtocolError& sync_protocol_error) override;
@@ skipping 26 matching lines @@
   FRIEND_TEST_ALL_PREFIXES(SyncSchedulerTest,
                            ServerConnectionChangeDuringBackoff);
   FRIEND_TEST_ALL_PREFIXES(SyncSchedulerTest,
                            ConnectionChangeCanaryPreemptedByNudge);
   FRIEND_TEST_ALL_PREFIXES(BackoffTriggersSyncSchedulerTest,
                            FailGetEncryptionKey);
   FRIEND_TEST_ALL_PREFIXES(SyncSchedulerTest, SuccessfulRetry);
   FRIEND_TEST_ALL_PREFIXES(SyncSchedulerTest, FailedRetry);
   FRIEND_TEST_ALL_PREFIXES(SyncSchedulerTest, ReceiveNewRetryDelay);
 
-  struct WaitInterval {
-    enum Mode {
-      // Uninitialized state, should not be set in practice.
-      UNKNOWN = -1,
-      // We enter a series of increasingly longer WaitIntervals if we experience
-      // repeated transient failures.  We retry at the end of each interval.
-      EXPONENTIAL_BACKOFF,
-      // A server-initiated throttled interval.  We do not allow any syncing
-      // during such an interval.
-      THROTTLED,
-    };
-    WaitInterval();
-    ~WaitInterval();
-    WaitInterval(Mode mode, base::TimeDelta length);
-
-    static const char* GetModeString(Mode mode);
-
-    Mode mode;
-    base::TimeDelta length;
-  };
-
   static const char* GetModeString(Mode mode);
 
   void SetDefaultNudgeDelay(base::TimeDelta delay_ms);
 
   // Invoke the syncer to perform a nudge job.
   void DoNudgeSyncCycleJob(JobPriority priority);
 
   // Invoke the syncer to perform a configuration job.
   void DoConfigurationSyncCycleJob(JobPriority priority);
 
   void DoClearServerDataSyncCycleJob(JobPriority priority);
 
   // Helper function for Do{Nudge,Configuration,Poll}SyncCycleJob.
   void HandleSuccess();
 
   // Helper function for Do{Nudge,Configuration,Poll}SyncCycleJob.
   void HandleFailure(const ModelNeutralState& model_neutral_state);
 
   // Invoke the Syncer to perform a poll job.
   void DoPollSyncCycleJob();
 
   // Helper function to calculate poll interval.
   base::TimeDelta GetPollInterval();
 
   // Adjusts the poll timer to account for new poll interval, and possibly
   // resets the poll interval, depedning on the flag's value.
   void AdjustPolling(PollAdjustType type);
 
-  // Helper to restart waiting with |wait_interval_|'s timer.
+  // Helper to restart pending_wakeup_timer_.
   void RestartWaiting();
 
   // Determines if we're allowed to contact the server right now.
   bool CanRunJobNow(JobPriority priority);
 
   // Determines if we're allowed to contact the server right now.
   bool CanRunNudgeJobNow(JobPriority priority);
 
   // If the scheduler's current state supports it, this will create a job based
   // on the passed in parameters and coalesce it with any other pending jobs,
   // then post a delayed task to run it.  It may also choose to drop the job or
   // save it for later, depending on the scheduler's current state.
   void ScheduleNudgeImpl(const base::TimeDelta& delay,
                          const tracked_objects::Location& nudge_location);
 
   // Helper to signal listeners about changed retry time.
   void NotifyRetryTime(base::Time retry_time);
 
   // Helper to signal listeners about changed throttled types.
   void NotifyThrottledTypesChanged(ModelTypeSet types);
 
+  // Helper to signal listeners about changed backed off types.
+  void NotifyBackedOffTypesChanged(ModelTypeSet types);
+
   // Looks for pending work and, if it finds any, run this work at "canary"
   // priority.
   void TryCanaryJob();
 
   // At the moment TrySyncCycleJob just posts call to TrySyncCycleJobImpl on
   // current thread. In the future it will request access token here.
   void TrySyncCycleJob();
   void TrySyncCycleJobImpl();
 
   // Transitions out of the THROTTLED WaitInterval then calls TryCanaryJob().
+  // This function is for global throttling.
   void Unthrottle();
 
-  // Called when a per-type throttling interval expires.
-  void TypeUnthrottle(base::TimeTicks unthrottle_time);
+  // Called when a per-type throttling or backing off interval expires.
+  void OnTypesUnblocked();
 
   // Runs a normal nudge job when the scheduled timer expires.
   void PerformDelayedNudge();
 
   // Attempts to exit EXPONENTIAL_BACKOFF by calling TryCanaryJob().
+  // This function is for global backoff.
   void ExponentialBackoffRetry();
 
   // Called when the root cause of the current connection error is fixed.
   void OnServerConnectionErrorFixed();
 
   // Creates a cycle for a poll and performs the sync.
   void PollTimerCallback();
 
   // Creates a cycle for a retry and performs the sync.
   void RetryTimerCallback();
 
-  // Returns the set of types that are enabled and not currently throttled.
-  ModelTypeSet GetEnabledAndUnthrottledTypes();
+  // Returns the set of types that are enabled and not currently throttled and
+  // backed off.
+  ModelTypeSet GetEnabledAndUnblockedTypes();
 
   // Called as we are started to broadcast an initial cycle snapshot
   // containing data like initial_sync_ended.  Important when the client starts
   // up and does not need to perform an initial sync.
   void SendInitialSnapshot();
 
   // This is used for histogramming and analysis of ScheduleNudge* APIs.
   // SyncScheduler is the ultimate choke-point for all such invocations (with
   // and without InvalidationState variants, all NudgeSources, etc) and as such
   // is the most flexible place to do this bookkeeping.
@@ skipping 20 matching lines @@
 
   // The mode of operation.
   Mode mode_;
 
   // Current wait state.  Null if we're not in backoff and not throttled.
   std::unique_ptr<WaitInterval> wait_interval_;
 
   std::unique_ptr<BackoffDelayProvider> delay_provider_;
 
   // The event that will wake us up.
+  // When the whole client got throttling or backoff, we will delay this timer
+  // as well.
   base::OneShotTimer pending_wakeup_timer_;
 
-  // An event that fires when data type throttling expires.
-  base::OneShotTimer type_unthrottle_timer_;
-
   // Storage for variables related to an in-progress configure request.  Note
   // that (mode_ != CONFIGURATION_MODE) \implies !pending_configure_params_.
   std::unique_ptr<ConfigurationParams> pending_configure_params_;
 
   std::unique_ptr<ClearParams> pending_clear_params_;
 
-  // If we have a nudge pending to run soon, it will be listed here.
-  base::TimeTicks scheduled_nudge_time_;
-
   // Keeps track of work that the syncer needs to handle.
   NudgeTracker nudge_tracker_;
 
   // Invoked to run through the sync cycle.
   std::unique_ptr<Syncer> syncer_;
 
   SyncCycleContext* cycle_context_;
 
   // A map tracking LOCAL NudgeSource invocations of ScheduleNudge* APIs,
   // organized by datatype. Each datatype that was part of the types requested
@@ skipping 29 matching lines @@
   // A second factory specially for weak_handle_this_, to allow the handle
   // to be const and alleviate threading concerns.
   base::WeakPtrFactory<SyncSchedulerImpl> weak_ptr_factory_for_weak_handle_;
 
   DISALLOW_COPY_AND_ASSIGN(SyncSchedulerImpl);
 };
 
 }  // namespace syncer
 
 #endif  // COMPONENTS_SYNC_ENGINE_IMPL_SYNC_SCHEDULER_IMPL_H_