Retry RPC deadlines when fetching GAE settings.

common/lazyslot/lazyslot.go

Index: common/lazyslot/lazyslot.go
diff --git a/common/lazyslot/lazyslot.go b/common/lazyslot/lazyslot.go
index 9cf59cee00592eb06abddff788c944c616bb7a43..4090faf0df66bde840df37e0b8b5061d76f4fd0a 100644
--- a/common/lazyslot/lazyslot.go
+++ b/common/lazyslot/lazyslot.go
@@ -17,6 +17,7 @@ import (
 	"golang.org/x/net/context"
 
 	"github.com/luci/luci-go/common/clock"
+	"github.com/luci/luci-go/common/retry"
 )
 
 // Value is what's stored in a Slot. It is treated as immutable value.
@@ -38,8 +39,10 @@ type Fetcher func(c context.Context, prev Value) (Value, error)
 // Only one goroutine will be busy refreshing, all others will see a slightly
 // stale copy of the value during the refresh.
 type Slot struct {
-	Fetcher Fetcher       // used to actually load the value on demand
-	Timeout time.Duration // how long to allow to fetch, 5 sec by default.
+	Fetcher       Fetcher        // used to actually load the value on demand
+	Timeout       time.Duration  // how long to allow to fetch, 15 sec by default.
+	RetryFactory  retry.Factory  // if non-nil, defines how to retry fetch errors
+	RetryCallback retry.Callback // called before retries, useful in tests
 
 	lock              sync.Mutex      // protects the guts below
 	current           *Value          // currently known value or nil if not fetched
@@ -56,9 +59,11 @@ type Slot struct {
 func (s *Slot) Get(c context.Context) (result Value, err error) {
 	// state is populate in the anonymous function below.
 	var state struct {
-		C         context.Context
-		Fetcher   Fetcher
-		PrevValue Value
+		C             context.Context
+		Fetcher       Fetcher
+		PrevValue     Value
+		RetryFactory  retry.Factory
+		RetryCallback retry.Callback
 	}
 
 	result, err, done := func() (result Value, err error, done bool) {
@@ -80,7 +85,7 @@ func (s *Slot) Get(c context.Context) (result Value, err error) {
 		// there's nothing to return yet. All goroutines would have to wait for this
 		// initial fetch to complete. They'll all block on s.lock.Lock() above.
 		if s.current == nil {
-			result, err = doFetch(c, s.Fetcher, Value{})
+			result, err = fetchWithRetries(s.makeFetcherCtx(c), s.Fetcher, s.RetryFactory, s.RetryCallback, Value{})
 			if err == nil {
 				s.current = &result
 			}
@@ -98,16 +103,14 @@ func (s *Slot) Get(c context.Context) (result Value, err error) {
 
 		// No one is fetching the value now, we should do it. Prepare a new context
 		// that will be used to do the fetch once lock is released.
-		timeout := 5 * time.Second
-		if s.Timeout != 0 {
-			timeout = s.Timeout
-		}
-		s.currentFetcherCtx, _ = context.WithTimeout(c, timeout)
+		s.currentFetcherCtx = s.makeFetcherCtx(c)
 
 		// Copy lock-protected guts into local variables before releasing the lock.
 		state.C = s.currentFetcherCtx
 		state.Fetcher = s.Fetcher
 		state.PrevValue = *s.current
+		state.RetryFactory = s.RetryFactory
+		state.RetryCallback = s.RetryCallback
 		return
 	}()
 	if done {
@@ -125,10 +128,37 @@ func (s *Slot) Get(c context.Context) (result Value, err error) {
 				s.current = &result
 			}
 		}()
-		return doFetch(state.C, state.Fetcher, state.PrevValue)
+		return fetchWithRetries(state.C, state.Fetcher, state.RetryFactory, state.RetryCallback, state.PrevValue)
 	}()
 }
 
+// makeFetcherCtx prepares a context to use for fetch operation.
+//
+// Must be called under the lock.
+func (s *Slot) makeFetcherCtx(c context.Context) context.Context {
+	timeout := 15 * time.Second
+	if s.Timeout != 0 {
+		timeout = s.Timeout
+	}
+	fetcherCtx, _ := clock.WithTimeout(c, timeout)
+	return fetcherCtx
+}
+
+// fetchWithRetries calls doFetch, retrying on errors.
+func fetchWithRetries(ctx context.Context, cb Fetcher, factory retry.Factory, retryCb retry.Callback, prev Value) (result Value, err error) {
+	if factory == nil {
+		return doFetch(ctx, cb, prev)
+	}
+	err = retry.Retry(clock.Tag(ctx, "retry"), factory, func() error {
+		result, err = doFetch(ctx, cb, prev)
+		return err
+	}, retryCb)
+	if err != nil {
+		result = Value{}
+	}
+	return
+}
+
 // doFetch calls fetcher callback and validates return value.
 func doFetch(ctx context.Context, cb Fetcher, prev Value) (result Value, err error) {
 	result, err = cb(ctx, prev)