Handle transient LogOut errors with retry and backoff.

components/signin/core/browser/gaia_cookie_manager_service.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/signin/core/browser/gaia_cookie_manager_service.h"
 
 #include <queue>
 #include <vector>
 
 #include "base/json/json_reader.h"
 #include "base/stl_util.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
 #include "base/values.h"
 #include "components/signin/core/browser/signin_metrics.h"
 #include "google_apis/gaia/gaia_auth_fetcher.h"
 #include "google_apis/gaia/gaia_auth_util.h"
 #include "google_apis/gaia/gaia_constants.h"
 #include "google_apis/gaia/gaia_urls.h"
 #include "google_apis/gaia/oauth2_token_service.h"
 #include "net/base/load_flags.h"
 #include "net/http/http_status_code.h"
 #include "net/url_request/url_fetcher.h"
 #include "net/url_request/url_fetcher_delegate.h"
 
 namespace {
 
-// In case of an error while fetching using the GaiaAuthFetcher, retry with
-// exponential backoff. Try up to 7 times within 15 minutes.
+// In case of an error while fetching using the GaiaAuthFetcher or URLFetcher,
+// retry with exponential backoff. Try up to 7 times within 15 minutes.
 const net::BackoffEntry::Policy kBackoffPolicy = {
   // Number of initial errors (in sequence) to ignore before applying
   // exponential back-off rules.
   0,
 
   // Initial delay for exponential backoff in ms.
   1000,
 
   // Factor by which the waiting time will be multiplied.
   3,
 
   // Fuzzing percentage. ex: 10% will spread requests randomly
   // between 90%-100% of the calculated time.
   0.2, // 20%
 
   // Maximum amount of time we are willing to delay our request in ms.
   1000 * 60 * 60 * 4, // 15 minutes.
 
   // Time to keep an entry from being discarded even when it
   // has no significant state, -1 to never discard.
   -1,
 
   // Don't use initial delay unless the last request was an error.
   false,
 };
 
-const int kMaxGaiaAuthFetcherRetries = 8;
+const int kMaxFetcherRetries = 8;
 
 bool IsTransientError(const GoogleServiceAuthError& error) {
   return error.state() == GoogleServiceAuthError::CONNECTION_FAILED ||
       error.state() == GoogleServiceAuthError::SERVICE_UNAVAILABLE ||
       error.state() == GoogleServiceAuthError::REQUEST_CANCELED;
 }
 
 enum GaiaCookieRequestType {
   ADD_ACCOUNT,
   LOG_OUT_ALL_ACCOUNTS,
@@ skipping 75 matching lines @@
 bool GaiaCookieManagerService::ExternalCcResultFetcher::IsRunning() {
   return helper_->gaia_auth_fetcher_ || fetchers_.size() > 0u;
 }
 
 void GaiaCookieManagerService::ExternalCcResultFetcher::TimeoutForTests() {
   Timeout();
 }
 
 void GaiaCookieManagerService::ExternalCcResultFetcher::
     OnGetCheckConnectionInfoSuccess(const std::string& data) {
-  helper_->gaia_auth_fetcher_backoff_.InformOfRequest(true);
+  helper_->fetcher_backoff_.InformOfRequest(true);
   gaia_auth_fetcher_timer_.Stop();
   scoped_ptr<base::Value> value(base::JSONReader::Read(data));
   const base::ListValue* list;
   if (!value || !value->GetAsList(&list)) {
     CleanupTransientState();
     GetCheckConnectionInfoCompleted(false);
     return;
   }
 
   // If there is nothing to check, terminate immediately.
@@ skipping 15 matching lines @@
         net::URLFetcher* fetcher = CreateFetcher(GURL(url));
         fetchers_[fetcher->GetOriginalURL()] = std::make_pair(token, fetcher);
         fetcher->Start();
       }
     }
   }
 }
 
 void GaiaCookieManagerService::ExternalCcResultFetcher::
     OnGetCheckConnectionInfoError(const GoogleServiceAuthError& error) {
-  if (++helper_->gaia_auth_fetcher_retries_ < kMaxGaiaAuthFetcherRetries &&
+  if (++helper_->fetcher_retries_ < kMaxFetcherRetries &&
       IsTransientError(error)) {
-    helper_->gaia_auth_fetcher_backoff_.InformOfRequest(false);
+    helper_->fetcher_backoff_.InformOfRequest(false);
     gaia_auth_fetcher_timer_.Start(
-        FROM_HERE, helper_->gaia_auth_fetcher_backoff_.GetTimeUntilRelease(),
+        FROM_HERE, helper_->fetcher_backoff_.GetTimeUntilRelease(),
         this, &GaiaCookieManagerService::ExternalCcResultFetcher::Start);
     return;
   }
 
   CleanupTransientState();
   GetCheckConnectionInfoCompleted(false);
 }
 
 net::URLFetcher*
 GaiaCookieManagerService::ExternalCcResultFetcher::CreateFetcher(
@@ skipping 72 matching lines @@
   helper_->StartFetchingMergeSession();
 }
 
 GaiaCookieManagerService::GaiaCookieManagerService(
     OAuth2TokenService* token_service,
     const std::string& source,
     SigninClient* signin_client)
     : token_service_(token_service),
       signin_client_(signin_client),
       external_cc_result_fetcher_(this),
-      gaia_auth_fetcher_backoff_(&kBackoffPolicy),
-      gaia_auth_fetcher_retries_(0),
+      fetcher_backoff_(&kBackoffPolicy),
+      fetcher_retries_(0),
       source_(source),
       external_cc_result_fetched_(false),
       list_accounts_fetched_once_(false) {
 }
 
 GaiaCookieManagerService::~GaiaCookieManagerService() {
   CancelAll();
   DCHECK(requests_.empty());
 }
 
@@ skipping 29 matching lines @@
   DCHECK(accounts);
   accounts->clear();
 
   // There is a fetch currently executing (the results being provided in the
   // parameter don't align with the fetches that have been started), or the list
   // of accounts haven't been fetched even once.
   if (!requests_.empty())
     return false;
 
   if (!list_accounts_fetched_once_) {
-    gaia_auth_fetcher_retries_ = 0;
+    fetcher_retries_ = 0;
     requests_.push_back(GaiaCookieRequest::CreateListAccountsRequest());
     StartFetchingListAccounts();
     return false;
   }
 
   accounts->assign(listed_accounts_.begin(), listed_accounts_.end());
   return true;
 }
 
 void GaiaCookieManagerService::LogOutAllAccounts() {
@@ skipping 29 matching lines @@
     // Remove all but the executing request. Re-add all requests being kept.
     if (requests_.size() > 1) {
       requests_.erase(requests_.begin() + 1, requests_.end());
       requests_.insert(
           requests_.end(), requests_to_keep.begin(), requests_to_keep.end());
     }
   }
 
   if (!log_out_queued) {
     requests_.push_back(GaiaCookieRequest::CreateLogOutRequest());
-    if (requests_.size() == 1)
+    if (requests_.size() == 1) {
+      fetcher_retries_ = 0;
       StartLogOutUrlFetch();
+    }
   }
 }
 
 void GaiaCookieManagerService::AddObserver(Observer* observer) {
   observer_list_.AddObserver(observer);
 }
 
 void GaiaCookieManagerService::RemoveObserver(Observer* observer) {
   observer_list_.RemoveObserver(observer);
 }
 
 void GaiaCookieManagerService::CancelAll() {
   VLOG(1) << "GaiaCookieManagerService::CancelAll";
   gaia_auth_fetcher_.reset();
   uber_token_fetcher_.reset();
   requests_.clear();
-  gaia_auth_fetcher_timer_.Stop();
+  fetcher_timer_.Stop();
 }
 
 // It is unknown if the cookie was changed because of processing initiated by
 // this class or other (such as the user clearing all cookies or a cookie being
 // evicted).
 void GaiaCookieManagerService::OnCookieChanged(
     const net::CanonicalCookie& cookie,
     bool removed) {
   DCHECK_EQ("APISID", cookie.Name());
   DCHECK_EQ(GaiaUrls::GetInstance()->google_url().host(), cookie.Domain());
-  gaia_auth_fetcher_retries_ = 0;
+  fetcher_retries_ = 0;

[Roger Tawa OOO till Jul 10th, 2015/04/27 21:23:05]

Is it safe to reset the retries count?  What if the current pending request is
in a retry state?  Maybe only do this at line 420 below?

[Mike Lerman, 2015/04/28 13:27:25]

Great catch - thanks! Done.

   if (requests_.empty()) {
     requests_.push_back(GaiaCookieRequest::CreateListAccountsRequest());
     StartFetchingListAccounts();
   } else {
     // Remove all pending ListAccount calls; for efficiency, only call
     // after all pending requests are processed.
     // Track requests to keep; all other unstarted requests will be removed.
     std::vector<GaiaCookieRequest> requests_to_keep;
 
     // Check all pending, non-executing requests.
@@ skipping 33 matching lines @@
   fetcher->SetRequestContext(signin_client_->GetURLRequestContext());
   fetcher->Start();
 }
 
 void GaiaCookieManagerService::OnUbertokenSuccess(
     const std::string& uber_token) {
   DCHECK(requests_.front().request_type() ==
       GaiaCookieRequestType::ADD_ACCOUNT);
   VLOG(1) << "GaiaCookieManagerService::OnUbertokenSuccess"
           << " account=" << requests_.front().account_id();
-  gaia_auth_fetcher_retries_ = 0;
+  fetcher_retries_ = 0;
   uber_token_ = uber_token;
 
   if (!external_cc_result_fetched_ &&
       !external_cc_result_fetcher_.IsRunning()) {
     external_cc_result_fetcher_.Start();
     return;
   }
 
   StartFetchingMergeSession();
 }
 
 void GaiaCookieManagerService::OnUbertokenFailure(
     const GoogleServiceAuthError& error) {
+  // Note that the UberToken fetcher already retries transient errors.
   VLOG(1) << "Failed to retrieve ubertoken"
           << " account=" << requests_.front().account_id()
           << " error=" << error.ToString();
   const std::string account_id = requests_.front().account_id();
   HandleNextRequest();
   SignalComplete(account_id, error);
 }
 
 void GaiaCookieManagerService::OnMergeSessionSuccess(const std::string& data) {
   VLOG(1) << "MergeSession successful account="
           << requests_.front().account_id();
   DCHECK(requests_.front().request_type() ==
          GaiaCookieRequestType::ADD_ACCOUNT);
   const std::string account_id = requests_.front().account_id();
   HandleNextRequest();
   SignalComplete(account_id, GoogleServiceAuthError::AuthErrorNone());
 
-  gaia_auth_fetcher_backoff_.InformOfRequest(true);
+  fetcher_backoff_.InformOfRequest(true);
   uber_token_ = std::string();
 }
 
 void GaiaCookieManagerService::OnMergeSessionFailure(
     const GoogleServiceAuthError& error) {
   DCHECK(requests_.front().request_type() ==
          GaiaCookieRequestType::ADD_ACCOUNT);
   VLOG(1) << "Failed MergeSession"
           << " account=" << requests_.front().account_id()
           << " error=" << error.ToString();
-  if (++gaia_auth_fetcher_retries_ < kMaxGaiaAuthFetcherRetries &&
-      IsTransientError(error)) {
-    gaia_auth_fetcher_backoff_.InformOfRequest(false);
-    gaia_auth_fetcher_timer_.Start(
-        FROM_HERE, gaia_auth_fetcher_backoff_.GetTimeUntilRelease(), this,
+  if (++fetcher_retries_ < kMaxFetcherRetries && IsTransientError(error)) {
+    fetcher_backoff_.InformOfRequest(false);
+    fetcher_timer_.Start(
+        FROM_HERE, fetcher_backoff_.GetTimeUntilRelease(), this,
         &GaiaCookieManagerService::StartFetchingMergeSession);
     return;
   }
 
   uber_token_ = std::string();
   const std::string account_id = requests_.front().account_id();
   HandleNextRequest();
   SignalComplete(account_id, error);
 }
 
 void GaiaCookieManagerService::OnListAccountsSuccess(const std::string& data) {
   VLOG(1) << "ListAccounts successful";
   DCHECK(requests_.front().request_type() ==
          GaiaCookieRequestType::LIST_ACCOUNTS);
-  gaia_auth_fetcher_backoff_.InformOfRequest(true);
+  fetcher_backoff_.InformOfRequest(true);
 
   if (!gaia::ParseListAccountsData(data, &listed_accounts_)) {
     listed_accounts_.clear();
     OnListAccountsFailure(GoogleServiceAuthError(
         GoogleServiceAuthError::UNEXPECTED_SERVICE_RESPONSE));
     return;
   }
 
   list_accounts_fetched_once_ = true;
   FOR_EACH_OBSERVER(Observer, observer_list_,
       OnGaiaAccountsInCookieUpdated(
           listed_accounts_,
           GoogleServiceAuthError(GoogleServiceAuthError::NONE)));
   HandleNextRequest();
 }
 
 void GaiaCookieManagerService::OnListAccountsFailure(
     const GoogleServiceAuthError& error) {
   VLOG(1) << "ListAccounts failed";
   DCHECK(requests_.front().request_type() ==
          GaiaCookieRequestType::LIST_ACCOUNTS);
-  if (++gaia_auth_fetcher_retries_ < kMaxGaiaAuthFetcherRetries &&
-      IsTransientError(error)) {
-    gaia_auth_fetcher_backoff_.InformOfRequest(false);
-    gaia_auth_fetcher_timer_.Start(
-        FROM_HERE, gaia_auth_fetcher_backoff_.GetTimeUntilRelease(), this,
+  if (++fetcher_retries_ < kMaxFetcherRetries && IsTransientError(error)) {
+    fetcher_backoff_.InformOfRequest(false);
+    fetcher_timer_.Start(
+        FROM_HERE, fetcher_backoff_.GetTimeUntilRelease(), this,
         &GaiaCookieManagerService::StartFetchingListAccounts);
     return;
   }
 
   FOR_EACH_OBSERVER(Observer, observer_list_,
       OnGaiaAccountsInCookieUpdated(listed_accounts_, error));
   HandleNextRequest();
 }
 
 void GaiaCookieManagerService::StartFetchingUbertoken() {
@@ skipping 20 matching lines @@
   gaia_auth_fetcher_.reset(
       new GaiaAuthFetcher(this, source_,
                           signin_client_->GetURLRequestContext()));
   gaia_auth_fetcher_->StartListAccounts();
 }
 
 void GaiaCookieManagerService::OnURLFetchComplete(
     const net::URLFetcher* source) {
   DCHECK(requests_.front().request_type() == GaiaCookieRequestType::LOG_OUT);
   VLOG(1) << "GaiaCookieManagerService::OnURLFetchComplete";
+
+  if ((!source->GetStatus().is_success() ||
+      source->GetResponseCode() != net::HTTP_OK) &&
+      ++fetcher_retries_ < kMaxFetcherRetries) {
+    fetcher_backoff_.InformOfRequest(false);
+    fetcher_timer_.Start(
+        FROM_HERE, fetcher_backoff_.GetTimeUntilRelease(), this,
+        &GaiaCookieManagerService::StartLogOutUrlFetch);
+    return;
+  }
+
+  fetcher_backoff_.InformOfRequest(true);
   HandleNextRequest();
 }
 
 void GaiaCookieManagerService::HandleNextRequest() {
   VLOG(1) << "GaiaCookieManagerService::HandleNextRequest";
   if (requests_.front().request_type() ==
       GaiaCookieRequestType::LIST_ACCOUNTS) {
     // This and any directly subsequent list accounts would return the same.
     while (!requests_.empty() && requests_.front().request_type() ==
            GaiaCookieRequestType::LIST_ACCOUNTS) {
       requests_.pop_front();
     }
   } else {
     // Pop the completed request.
     requests_.pop_front();
   }
 
   gaia_auth_fetcher_.reset();
-  gaia_auth_fetcher_retries_ = 0;
+  fetcher_retries_ = 0;
   if (requests_.empty()) {
     VLOG(1) << "GaiaCookieManagerService::HandleNextRequest: no more";
     uber_token_fetcher_.reset();
   } else {
     switch (requests_.front().request_type()) {
       case GaiaCookieRequestType::ADD_ACCOUNT:
         StartFetchingUbertoken();
         break;
       case GaiaCookieRequestType::LOG_OUT:
         StartLogOutUrlFetch();
         break;
       case GaiaCookieRequestType::LIST_ACCOUNTS:
         uber_token_fetcher_.reset();
         StartFetchingListAccounts();
         break;
     };
   }
 }