Retry RPC deadlines when fetching GAE settings.

server/settings/settings.go

Index: server/settings/settings.go
diff --git a/server/settings/settings.go b/server/settings/settings.go
index bd05fa0503978292d68f0c79694076e15adfca7a..717d9e6ede45ad5ba16952ec67da49582728868a 100644
--- a/server/settings/settings.go
+++ b/server/settings/settings.go
@@ -19,7 +19,9 @@ import (
 
 	"golang.org/x/net/context"
 
+	"github.com/luci/luci-go/common/clock"
 	"github.com/luci/luci-go/common/lazyslot"
+	"github.com/luci/luci-go/common/retry"
 )
 
 var (
@@ -116,7 +118,10 @@ func New(storage Storage) *Settings {
 	return &Settings{
 		storage: storage,
 		values: lazyslot.Slot{
+			RetryFactory: retry.TransientOnly(retry.Default), // retry transient errors
+			Timeout:      15 * time.Second,                   // retry for 15 sec at most
 			Fetcher: func(c context.Context, _ lazyslot.Value) (lazyslot.Value, error) {
+				c, _ = clock.WithTimeout(c, 2*time.Second) // trigger a retry after 2 sec RPC timeout

[dnj, 2016/03/01 02:21:40]

WDYT about moving the retries into the Fetcher instead of lazySlot? It keeps
lazySlot high-level ("I call this function!") and stops you from having to pass
parameters in and (effectively) back out.

[Vadim Sh., 2016/03/02 22:52:04]

Done.

 				bundle, err := storage.FetchAllSettings(c)
 				if err != nil {
 					return lazyslot.Value{}, err