Retry RPC deadlines when fetching GAE settings.

server/settings/settings.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Package settings implements storage for infrequently changing global
 // settings.
 //
 // Settings are represented as (key, value) pairs, where value is JSON
 // serializable struct. Settings are cached internally in the process memory to
 // avoid hitting the storage all the time.
 package settings
 
 import (
         "encoding/json"
         "errors"
         "reflect"
         "sync"
         "time"
 
         "golang.org/x/net/context"
 
+        "github.com/luci/luci-go/common/clock"
         "github.com/luci/luci-go/common/lazyslot"
+        "github.com/luci/luci-go/common/retry"
 )
 
 var (
         // ErrNoSettings can be returned by Get and Set on fatal errors.
         ErrNoSettings = errors.New("settings: settings are not available")
 
         // ErrBadType is returned if Get(...) receives unexpected type.
         ErrBadType = errors.New("settings: bad type")
 )
 
@@ skipping 76 matching lines @@
         storage Storage       // used to load and save settings
         values  lazyslot.Slot // cached settings
 }
 
 // New creates new Settings object that uses given Storage to fetch and save
 // settings.
 func New(storage Storage) *Settings {
         return &Settings{
                 storage: storage,
                 values: lazyslot.Slot{
-                        Fetcher: func(c context.Context, _ lazyslot.Value) (lazyslot.Value, error) {
-                                bundle, err := storage.FetchAllSettings(c)
+                        Timeout: 15 * time.Second, // retry for 15 sec at most
+                        Fetcher: func(c context.Context, _ lazyslot.Value) (result lazyslot.Value, err error) {
+                                err = retry.Retry(c, retry.TransientOnly(retry.Default), func() error {
+                                        ctx, _ := clock.WithTimeout(c, 2*time.Second) // trigger a retry after 2 sec RPC timeout
+                                        result, err = attemptToFetchSettings(ctx, storage)
+                                        return err
+                                }, nil)
                                 if err != nil {
-                                        return lazyslot.Value{}, err
+                                        result = lazyslot.Value{}
                                 }
-                                return lazyslot.Value{
-                                        Value:      bundle,
-                                        Expiration: bundle.Exp,
-                                }, nil
+                                return
                         },
                 },
         }
 }
 
+// attemptToFetchSettings is called in a retry loop when loading settings.
+func attemptToFetchSettings(c context.Context, storage Storage) (lazyslot.Value, error) {
+        bundle, err := storage.FetchAllSettings(c)
+        if err != nil {
+                return lazyslot.Value{}, err
+        }
+        return lazyslot.Value{
+                Value:      bundle,
+                Expiration: bundle.Exp,
+        }, nil
+}
+
 // GetStorage returns underlying Storage instance.
 func (s *Settings) GetStorage() Storage {
         return s.storage
 }
 
 // Get returns setting value (possibly cached) for the given key.
 //
 // It will be deserialized into the supplied value. Caller is responsible
 // to pass correct type and pass same type to all calls. If the setting is not
 // set returns ErrNoSettings.
@@ skipping 21 matching lines @@
 //
 // New settings will apply only when existing in-memory cache expires.
 // In particular, Get() right after Set() may still return old value.
 func (s *Settings) Set(c context.Context, key string, value interface{}, who, why string) error {
         blob, err := json.Marshal(value)
         if err != nil {
                 return err
         }
         return s.storage.UpdateSetting(c, key, json.RawMessage(blob), who, why)
 }