Retry RPC deadlines when fetching GAE settings.

server/settings/settings.go

Index: server/settings/settings.go
diff --git a/server/settings/settings.go b/server/settings/settings.go
index bd05fa0503978292d68f0c79694076e15adfca7a..678c42cf4926a13aac0a32b10701f60f136b365b 100644
--- a/server/settings/settings.go
+++ b/server/settings/settings.go
@@ -19,7 +19,9 @@ import (
 
 	"golang.org/x/net/context"
 
+	"github.com/luci/luci-go/common/clock"
 	"github.com/luci/luci-go/common/lazyslot"
+	"github.com/luci/luci-go/common/retry"
 )
 
 var (
@@ -116,20 +118,34 @@ func New(storage Storage) *Settings {
 	return &Settings{
 		storage: storage,
 		values: lazyslot.Slot{
-			Fetcher: func(c context.Context, _ lazyslot.Value) (lazyslot.Value, error) {
-				bundle, err := storage.FetchAllSettings(c)
+			Timeout: 15 * time.Second, // retry for 15 sec at most
+			Fetcher: func(c context.Context, _ lazyslot.Value) (result lazyslot.Value, err error) {
+				err = retry.Retry(c, retry.TransientOnly(retry.Default), func() error {
+					ctx, _ := clock.WithTimeout(c, 2*time.Second) // trigger a retry after 2 sec RPC timeout
+					result, err = attemptToFetchSettings(ctx, storage)
+					return err
+				}, nil)
 				if err != nil {
-					return lazyslot.Value{}, err
+					result = lazyslot.Value{}
 				}
-				return lazyslot.Value{
-					Value:      bundle,
-					Expiration: bundle.Exp,
-				}, nil
+				return
 			},
 		},
 	}
 }
 
+// attemptToFetchSettings is called in a retry loop when loading settings.
+func attemptToFetchSettings(c context.Context, storage Storage) (lazyslot.Value, error) {
+	bundle, err := storage.FetchAllSettings(c)
+	if err != nil {
+		return lazyslot.Value{}, err
+	}
+	return lazyslot.Value{
+		Value:      bundle,
+		Expiration: bundle.Exp,
+	}, nil
+}
+
 // GetStorage returns underlying Storage instance.
 func (s *Settings) GetStorage() Storage {
 	return s.storage