Retry RPC deadlines when fetching GAE settings.

server/settings/settings.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Package settings implements storage for infrequently changing global
 // settings.
 //
 // Settings are represented as (key, value) pairs, where value is JSON
 // serializable struct. Settings are cached internally in the process memory to
 // avoid hitting the storage all the time.
 package settings
 
 import (
         "encoding/json"
         "errors"
         "reflect"
         "sync"
         "time"
 
         "golang.org/x/net/context"
 
+        "github.com/luci/luci-go/common/clock"
         "github.com/luci/luci-go/common/lazyslot"
+        "github.com/luci/luci-go/common/retry"
 )
 
 var (
         // ErrNoSettings can be returned by Get and Set on fatal errors.
         ErrNoSettings = errors.New("settings: settings are not available")
 
         // ErrBadType is returned if Get(...) receives unexpected type.
         ErrBadType = errors.New("settings: bad type")
 )
 
@@ skipping 76 matching lines @@
         storage Storage       // used to load and save settings
         values  lazyslot.Slot // cached settings
 }
 
 // New creates new Settings object that uses given Storage to fetch and save
 // settings.
 func New(storage Storage) *Settings {
         return &Settings{
                 storage: storage,
                 values: lazyslot.Slot{
+                        RetryFactory: retry.TransientOnly(retry.Default), // retry transient errors
+                        Timeout:      15 * time.Second,                   // retry for 15 sec at most
                         Fetcher: func(c context.Context, _ lazyslot.Value) (lazyslot.Value, error) {
+                                c, _ = clock.WithTimeout(c, 2*time.Second) // trigger a retry after 2 sec RPC timeout

[dnj, 2016/03/01 02:21:40]

WDYT about moving the retries into the Fetcher instead of lazySlot? It keeps
lazySlot high-level ("I call this function!") and stops you from having to pass
parameters in and (effectively) back out.

[Vadim Sh., 2016/03/02 22:52:04]

Done.

                                 bundle, err := storage.FetchAllSettings(c)
                                 if err != nil {
                                         return lazyslot.Value{}, err
                                 }
                                 return lazyslot.Value{
                                         Value:      bundle,
                                         Expiration: bundle.Exp,
                                 }, nil
                         },
                 },
@@ skipping 34 matching lines @@
 //
 // New settings will apply only when existing in-memory cache expires.
 // In particular, Get() right after Set() may still return old value.
 func (s *Settings) Set(c context.Context, key string, value interface{}, who, why string) error {
         blob, err := json.Marshal(value)
         if err != nil {
                 return err
         }
         return s.storage.UpdateSetting(c, key, json.RawMessage(blob), who, why)
 }