[Sync] Sync client should to exponential backoff when receive partial failure

components/sync/engine_impl/sync_scheduler_impl_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/sync/engine_impl/sync_scheduler_impl.h"
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <vector>
@@ skipping 151 matching lines @@
         syncer_);
     scheduler_->SetDefaultNudgeDelay(default_delay());
   }
 
   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
   MockSyncer* syncer() { return syncer_; }
   MockDelayProvider* delay() { return delay_; }
   MockConnectionManager* connection() { return connection_.get(); }
   TimeDelta default_delay() { return TimeDelta::FromSeconds(0); }
+  TimeDelta long_delay() { return TimeDelta::FromSeconds(60); }
   TimeDelta timeout() { return TestTimeouts::action_timeout(); }
 
   void TearDown() override {
     PumpLoop();
     scheduler_.reset();
     PumpLoop();
     test_user_share_.TearDown();
   }
 
   void AnalyzePollRun(const SyncShareTimes& times,
@@ skipping 37 matching lines @@
   }
 
   void UseMockDelayProvider() {
     delay_ = new MockDelayProvider();
     scheduler_->delay_provider_.reset(delay_);
   }
 
   SyncCycleContext* context() { return context_.get(); }
 
   ModelTypeSet GetThrottledTypes() {
-    return scheduler_->nudge_tracker_.GetThrottledTypes();
+    ModelTypeSet throttled_types;
+    ModelTypeSet blocked_types = scheduler_->nudge_tracker_.GetBlockedTypes();
+    for (ModelTypeSet::Iterator type_it = blocked_types.First(); type_it.Good();
+         type_it.Inc()) {
+      if (scheduler_->nudge_tracker_.GetTypeBlockingMode(type_it.Get()) ==
+          WaitInterval::THROTTLED) {
+        throttled_types.Put(type_it.Get());
+      }
+    }
+    return throttled_types;
+  }
+
+  ModelTypeSet GetBackedOffTypes() {
+    ModelTypeSet backed_off_types;
+    ModelTypeSet blocked_types = scheduler_->nudge_tracker_.GetBlockedTypes();
+    for (ModelTypeSet::Iterator type_it = blocked_types.First(); type_it.Good();
+         type_it.Inc()) {
+      if (scheduler_->nudge_tracker_.GetTypeBlockingMode(type_it.Get()) ==
+          WaitInterval::EXPONENTIAL_BACKOFF) {
+        backed_off_types.Put(type_it.Get());
+      }
+    }
+    return backed_off_types;
+  }
+
+  bool IsAnyTypeBlocked() {
+    return scheduler_->nudge_tracker_.IsAnyTypeBlocked();
   }
 
   base::TimeDelta GetRetryTimerDelay() {
     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
     return scheduler_->retry_timer_.GetCurrentDelay();
   }
 
   static std::unique_ptr<InvalidationInterface> BuildInvalidation(
       int64_t version,
       const std::string& payload) {
     return MockInvalidation::Build(version, payload);
   }
 
+  base::TimeDelta GetTypeBlockingTime(ModelType type) {
+    NudgeTracker::TypeTrackerMap::const_iterator tracker_it =
+        scheduler_->nudge_tracker_.type_trackers_.find(type);
+    DCHECK(tracker_it != scheduler_->nudge_tracker_.type_trackers_.end());
+    return tracker_it->second->GetTimeUntilUnblock();

[Nicolas Zea, 2016/11/14 21:57:15]

I think it would be better to query the wait interval time directly, rather than
time until unblock.

[Gang Wu, 2016/11/15 01:59:32]

Done.

+  }
+
+  void SetTypeBlockingMode(ModelType type, WaitInterval::BlockingMode mode) {
+    NudgeTracker::TypeTrackerMap::const_iterator tracker_it =
+        scheduler_->nudge_tracker_.type_trackers_.find(type);
+    DCHECK(tracker_it != scheduler_->nudge_tracker_.type_trackers_.end());
+    DCHECK(tracker_it->second->wait_interval_.get());
+    tracker_it->second->wait_interval_->mode = mode;
+  }
+
  private:
   syncable::Directory* directory() {
     return test_user_share_.user_share()->directory.get();
   }
 
   base::MessageLoop loop_;
   TestUserShare test_user_share_;
   CancelationSignal cancelation_signal_;
   std::unique_ptr<MockConnectionManager> connection_;
   std::unique_ptr<ModelTypeRegistry> model_type_registry_;
@@ skipping 545 matching lines @@
   EXPECT_EQ(1, retry_counter.times_called());
   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 
   RunLoop();
   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 
   StopSyncScheduler();
 }
 
 TEST_F(SyncSchedulerImplTest, TypeThrottlingBlocksNudge) {
-  UseMockDelayProvider();
-  EXPECT_CALL(*delay(), GetDelay(_)).WillRepeatedly(Return(default_delay()));
-
   TimeDelta poll(TimeDelta::FromDays(1));
   TimeDelta throttle1(TimeDelta::FromSeconds(60));
   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 
   const ModelTypeSet types(THEMES);
 
   ::testing::InSequence seq;
   EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
       .WillOnce(
           DoAll(WithArg<2>(test_util::SimulateTypesThrottled(types, throttle1)),
-                Return(false)))
+                Return(true)))
       .RetiresOnSaturation();
 
   StartSyncScheduler(base::Time());
   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
   PumpLoop();  // To get PerformDelayedNudge called.
   PumpLoop();  // To get TrySyncCycleJob called
   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 
   // This won't cause a sync cycle because the types are throttled.
   scheduler()->ScheduleLocalNudge(types, FROM_HERE);
   PumpLoop();
 
   StopSyncScheduler();
 }
 
+TEST_F(SyncSchedulerImplTest, TypeBackingOffBlocksNudge) {
+  UseMockDelayProvider();
+  EXPECT_CALL(*delay(), GetDelay(_)).WillRepeatedly(Return(long_delay()));
+
+  TimeDelta poll(TimeDelta::FromDays(1));
+  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
+
+  const ModelTypeSet types(THEMES);
+
+  ::testing::InSequence seq;
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillOnce(DoAll(WithArg<2>(test_util::SimulatePartialFailure(types)),
+                      Return(true)))
+      .RetiresOnSaturation();
+
+  StartSyncScheduler(base::Time());
+  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
+  PumpLoop();  // To get PerformDelayedNudge called.
+  PumpLoop();  // To get TrySyncCycleJob called
+  EXPECT_TRUE(GetBackedOffTypes().HasAll(types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
+
+  // This won't cause a sync cycle because the types are backed off.
+  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
+  PumpLoop();
+
+  StopSyncScheduler();
+}
+
+TEST_F(SyncSchedulerImplTest, TypeBackingOffWillExpire) {
+  UseMockDelayProvider();
+  EXPECT_CALL(*delay(), GetDelay(_)).WillRepeatedly(Return(default_delay()));
+
+  TimeDelta poll(TimeDelta::FromDays(1));
+  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
+
+  const ModelTypeSet types(THEMES);
+
+  ::testing::InSequence seq;
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillOnce(DoAll(WithArg<2>(test_util::SimulatePartialFailure(types)),
+                      Return(true)))
+      .RetiresOnSaturation();
+
+  StartSyncScheduler(base::Time());
+  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
+  PumpLoop();  // To get PerformDelayedNudge called.
+  PumpLoop();  // To get TrySyncCycleJob called
+  EXPECT_TRUE(GetBackedOffTypes().HasAll(types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
+
+  SyncShareTimes times;
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillRepeatedly(DoAll(Invoke(test_util::SimulateNormalSuccess),
+                            RecordSyncShare(&times, true)));
+  PumpLoop();  // To get PerformDelayedNudge called.
+  PumpLoop();  // To get TrySyncCycleJob called
+  EXPECT_FALSE(IsAnyTypeBlocked());
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
+
+  StopSyncScheduler();
+}
+
+TEST_F(SyncSchedulerImplTest, TypeBackingOffAndThrottling) {
+  UseMockDelayProvider();
+  EXPECT_CALL(*delay(), GetDelay(_)).WillRepeatedly(Return(long_delay()));
+
+  TimeDelta poll(TimeDelta::FromDays(1));
+  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
+
+  const ModelTypeSet types(THEMES);
+
+  ::testing::InSequence seq;
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillOnce(DoAll(WithArg<2>(test_util::SimulatePartialFailure(types)),
+                      Return(true)))
+      .RetiresOnSaturation();
+
+  StartSyncScheduler(base::Time());
+  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
+  PumpLoop();  // To get PerformDelayedNudge called.
+  PumpLoop();  // To get TrySyncCycleJob called
+  EXPECT_TRUE(GetBackedOffTypes().HasAll(types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
+
+  TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
+
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillOnce(DoAll(WithArg<2>(test_util::SimulateThrottled(throttle1)),
+                      Return(false)))
+      .RetiresOnSaturation();
+
+  // Sync still can throttle.
+  const ModelTypeSet unbacked_off_types(TYPED_URLS);
+  scheduler()->ScheduleLocalNudge(unbacked_off_types, FROM_HERE);
+  PumpLoop();  // TO get TypesUnblock called.
+  PumpLoop();  // To get TrySyncCycleJob called.
+
+  EXPECT_TRUE(GetBackedOffTypes().HasAll(types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
+
+  StopSyncScheduler();
+}
+
+TEST_F(SyncSchedulerImplTest, TypeThrottlingBackingOffBlocksNudge) {
+  UseMockDelayProvider();
+  EXPECT_CALL(*delay(), GetDelay(_)).WillRepeatedly(Return(long_delay()));
+
+  TimeDelta poll(TimeDelta::FromDays(1));
+  TimeDelta throttle(TimeDelta::FromSeconds(60));
+  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
+
+  const ModelTypeSet throttled_types(THEMES);
+
+  ::testing::InSequence seq;
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillOnce(DoAll(WithArg<2>(test_util::SimulateTypesThrottled(
+                          throttled_types, throttle)),
+                      Return(true)))
+      .RetiresOnSaturation();
+
+  StartSyncScheduler(base::Time());
+  scheduler()->ScheduleLocalNudge(throttled_types, FROM_HERE);
+  PumpLoop();  // To get PerformDelayedNudge called.
+  PumpLoop();  // To get TrySyncCycleJob called
+
+  const ModelTypeSet backed_off_types(TYPED_URLS);
+
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillOnce(
+          DoAll(WithArg<2>(test_util::SimulatePartialFailure(backed_off_types)),
+                Return(true)))
+      .RetiresOnSaturation();
+
+  scheduler()->ScheduleLocalNudge(backed_off_types, FROM_HERE);
+
+  PumpLoop();  // To get PerformDelayedNudge called.
+  PumpLoop();  // To get TrySyncCycleJob called
+
+  EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
+  EXPECT_TRUE(GetBackedOffTypes().HasAll(backed_off_types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
+
+  // This won't cause a sync cycle because the types are throttled or backed
+  // off.
+  scheduler()->ScheduleLocalNudge(Union(throttled_types, backed_off_types),
+                                  FROM_HERE);
+  PumpLoop();
+
+  StopSyncScheduler();
+}
+
 TEST_F(SyncSchedulerImplTest, TypeThrottlingDoesBlockOtherSources) {
   UseMockDelayProvider();
   EXPECT_CALL(*delay(), GetDelay(_)).WillRepeatedly(Return(default_delay()));
 
   SyncShareTimes times;
   TimeDelta poll(TimeDelta::FromDays(1));
   TimeDelta throttle1(TimeDelta::FromSeconds(60));
   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 
   const ModelTypeSet throttled_types(THEMES);
   const ModelTypeSet unthrottled_types(PREFERENCES);
 
   ::testing::InSequence seq;
   EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
       .WillOnce(DoAll(WithArg<2>(test_util::SimulateTypesThrottled(
                           throttled_types, throttle1)),
-                      Return(false)))
+                      Return(true)))
       .RetiresOnSaturation();
 
   StartSyncScheduler(base::Time());
   scheduler()->ScheduleLocalNudge(throttled_types, FROM_HERE);
   PumpLoop();  // To get PerformDelayedNudge called.
   PumpLoop();  // To get TrySyncCycleJob called
   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 
   // Ignore invalidations for throttled types.
   scheduler()->ScheduleInvalidationNudge(THEMES, BuildInvalidation(10, "test"),
                                          FROM_HERE);
   PumpLoop();
 
   // Ignore refresh requests for throttled types.
   scheduler()->ScheduleLocalRefreshRequest(throttled_types, FROM_HERE);
   PumpLoop();
 
   Mock::VerifyAndClearExpectations(syncer());
 
   // Local nudges for non-throttled types will trigger a sync.
   EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
       .WillRepeatedly(DoAll(Invoke(test_util::SimulateNormalSuccess),
                             RecordSyncShare(&times, true)));
   scheduler()->ScheduleLocalNudge(unthrottled_types, FROM_HERE);
   RunLoop();
   Mock::VerifyAndClearExpectations(syncer());
 
   StopSyncScheduler();
 }
 
+TEST_F(SyncSchedulerImplTest, TypeBackingOffDoesBlockOtherSources) {
+  UseMockDelayProvider();
+  EXPECT_CALL(*delay(), GetDelay(_)).WillRepeatedly(Return(long_delay()));
+
+  SyncShareTimes times;
+  TimeDelta poll(TimeDelta::FromDays(1));
+  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
+
+  const ModelTypeSet backed_off_types(THEMES);
+  const ModelTypeSet unbacked_off_types(PREFERENCES);
+
+  ::testing::InSequence seq;
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillOnce(
+          DoAll(WithArg<2>(test_util::SimulatePartialFailure(backed_off_types)),
+                Return(true)))
+      .RetiresOnSaturation();
+
+  StartSyncScheduler(base::Time());
+  scheduler()->ScheduleLocalNudge(backed_off_types, FROM_HERE);
+  PumpLoop();  // To get PerformDelayedNudge called.
+  PumpLoop();  // To get TrySyncCycleJob called
+  EXPECT_TRUE(GetBackedOffTypes().HasAll(backed_off_types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
+
+  // Ignore invalidations for backed off types.
+  scheduler()->ScheduleInvalidationNudge(THEMES, BuildInvalidation(10, "test"),
+                                         FROM_HERE);
+  PumpLoop();
+
+  // Ignore refresh requests for backed off types.
+  scheduler()->ScheduleLocalRefreshRequest(backed_off_types, FROM_HERE);
+  PumpLoop();
+
+  Mock::VerifyAndClearExpectations(syncer());
+
+  // Local nudges for non-backed off types will trigger a sync.
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillRepeatedly(DoAll(Invoke(test_util::SimulateNormalSuccess),
+                            RecordSyncShare(&times, true)));
+  scheduler()->ScheduleLocalNudge(unbacked_off_types, FROM_HERE);
+  RunLoop();
+  Mock::VerifyAndClearExpectations(syncer());
+
+  StopSyncScheduler();
+}
+
 // Test nudges / polls don't run in config mode and config tasks do.
 TEST_F(SyncSchedulerImplTest, ConfigurationMode) {
   TimeDelta poll(TimeDelta::FromMilliseconds(15));
   SyncShareTimes times;
   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 
   StartSyncConfiguration();
 
   const ModelTypeSet nudge_types(TYPED_URLS);
   scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
@@ skipping 538 matching lines @@
   ASSERT_EQ(0, success_counter.times_called());
   ASSERT_TRUE(scheduler()->IsBackingOff());
 
   // Now succeed.
   connection()->SetServerReachable();
   PumpLoopFor(2 * delta);
   ASSERT_EQ(1, success_counter.times_called());
   ASSERT_FALSE(scheduler()->IsBackingOff());
 }
 
+TEST_F(SyncSchedulerImplTest, PartialFailureWillExponentialBackoff) {
+  TimeDelta poll(TimeDelta::FromDays(1));
+  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
+
+  const ModelTypeSet types(THEMES);
+
+  ::testing::InSequence seq;
+  EXPECT_CALL(*syncer(), NormalSyncShare(_, _, _))
+      .WillRepeatedly(DoAll(
+          WithArg<2>(test_util::SimulatePartialFailure(types)), Return(true)))
+      .RetiresOnSaturation();
+
+  StartSyncScheduler(base::Time());
+  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
+  PumpLoop();  // To get PerformDelayedNudge called.
+  PumpLoop();  // To get TrySyncCycleJob called
+  EXPECT_TRUE(GetBackedOffTypes().HasAll(types));
+  EXPECT_FALSE(scheduler()->IsBackingOff());
+  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
+  base::TimeDelta first_blocking_time = GetTypeBlockingTime(THEMES);
+
+  SetTypeBlockingMode(THEMES, WaitInterval::EXPONENTIAL_BACKOFF_RETRYING);
+  // This won't cause a sync cycle because the types are backed off.
+  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
+  PumpLoop();
+  PumpLoop();
+  base::TimeDelta second_blocking_time = GetTypeBlockingTime(THEMES);
+
+  // The Exponential backoff should between previous backoff 1.5 and 2.5 times.

[Nicolas Zea, 2016/11/14 21:57:15]

"should between" -> "should be between"

[Gang Wu, 2016/11/15 01:59:32]

Done.

+  // But |GetTypeBlockingTime| is to get time delta from base::Time::Now() to
+  // next un-backoff, so we enlarge our scope from 1.5-2.5 to 1.4-2.6 to avoid
+  // delay between "set backoff" and |GetTypeBlockingTime|.
+  EXPECT_LE(first_blocking_time * 1.4, second_blocking_time);

[Nicolas Zea, 2016/11/14 21:57:15]

If you query the interval time, rather than time until unblock, you should be
better able to verify that it has grown exponentially.

[Gang Wu, 2016/11/15 01:59:32]

Done.

+  EXPECT_GE(first_blocking_time * 2.6, second_blocking_time);
+
+  StopSyncScheduler();
+}
+
 }  // namespace syncer