Retry RPC deadlines when fetching GAE settings.

common/lazyslot/lazyslot.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Package lazyslot implements a caching scheme for globally shared objects that
 // take significant time to refresh.
 //
 // The defining property of the implementation is that only one goroutine
 // (can be background one) will block when refreshing such object, while all
 // others will use a slightly stale cached copy.
 package lazyslot
 
 import (
         "fmt"
         "sync"
         "time"
 
         "golang.org/x/net/context"
 
         "github.com/luci/luci-go/common/clock"
+        "github.com/luci/luci-go/common/retry"
 )
 
 // Value is what's stored in a Slot. It is treated as immutable value.
 type Value struct {
         // Value is whatever fetcher returned.
         Value interface{}
         // Expiration is time when this value expires and should be refetched.
         Expiration time.Time
 }
 
 // Fetcher knows how to load a new value.
 //
 // If it returns no errors, it MUST return non-nil Value.Value or Slot.Get will
 // return error.
 //
 // Panics inside Fetcher will be caught and converted to errors.
 type Fetcher func(c context.Context, prev Value) (Value, error)
 
 // Slot holds a cached Value and refreshes it when it expires.
 //
 // Only one goroutine will be busy refreshing, all others will see a slightly
 // stale copy of the value during the refresh.
 type Slot struct {
-        Fetcher Fetcher       // used to actually load the value on demand
-        Timeout time.Duration // how long to allow to fetch, 5 sec by default.
-        Async   bool          // if true do fetches in background goroutine
+        Fetcher       Fetcher        // used to actually load the value on demand
+        Timeout       time.Duration  // how long to allow to fetch, 15 sec by default.
+        Async         bool           // if true do fetches in background goroutine
+        RetryFactory  retry.Factory  // if non-nil, defines how to retry fetch errors
+        RetryCallback retry.Callback // called before retries, useful in tests

[iannucci, 2016/03/01 00:44:21]

Is this still used?

[Vadim Sh., 2016/03/01 01:24:05]

No, but it can be useful in general (e.g. to plug in logging for retries), so I
kept it.

 
         lock              sync.Mutex      // protects the guts below
         current           *Value          // currently known value or nil if not fetched
         currentFetcherCtx context.Context // non-nil if some goroutine is fetching now
 }
 
 // Get returns stored value if it is still fresh.
 //
 // It may return slightly stale copy if some other goroutine is fetching a new
 // copy now. If there's no cached copy at all, blocks until it is retrieved
@@ skipping 14 matching lines @@
         if s.current != nil && now.Before(s.current.Expiration) {
                 result = *s.current
                 s.lock.Unlock()
                 return
         }
 
         // Fetching the value for the first time ever? Do it under the lock because
         // there's nothing to return yet. All goroutines would have to wait for this
         // initial fetch to complete. They'll all block on s.lock.Lock() above.
         if s.current == nil {
-                result, err = doFetchNoPanic(c, s.Fetcher, Value{})
+                result, err = fetchWithRetries(s.makeFetcherCtx(c), s.Fetcher, s.RetryFactory, s.RetryCallback, Value{})
                 if err == nil {
                         s.current = &result
                 }
                 s.lock.Unlock()
                 return
         }
 
         // We have a cached copy but it has expired. Maybe some other goroutine is
         // fetching it already? Returns the cached stale copy if so.
         if s.currentFetcherCtx != nil {
                 result = *s.current
                 s.lock.Unlock()
                 return
         }
 
         // No one is fetching the value now, we should do it. Prepare new context
         // that will be used to do the fetch once lock is released.
-        timeout := 5 * time.Second
-        if s.Timeout != 0 {
-                timeout = s.Timeout
-        }
-        s.currentFetcherCtx, _ = context.WithTimeout(c, timeout)
+        s.currentFetcherCtx = s.makeFetcherCtx(c)
 
         // Copy lock-protected guts into local variables before releasing the lock.
         currentFetcherCtx := s.currentFetcherCtx
         fetchCb := s.Fetcher
         prevVal := *s.current
         async := s.Async
+        retryFactory := s.RetryFactory
+        retryCallback := s.RetryCallback
 
         // Release the lock to allow other goroutines to grab stale copy while fetch
         // is in the progress.
         s.lock.Unlock()
 
         // fetch finishes the fetch and updates cached value.
         fetch := func() (result Value, err error) {
                 defer func() {
                         s.lock.Lock()
                         defer s.lock.Unlock()
                         s.currentFetcherCtx = nil
                         if err == nil {
                                 s.current = &result
                         }
                 }()
-                return doFetchNoPanic(currentFetcherCtx, fetchCb, prevVal)
+                return fetchWithRetries(currentFetcherCtx, fetchCb, retryFactory, retryCallback, prevVal)
         }
 
         if async {
                 // Start async fetch and return stale copy while it is running.
                 go fetch()
                 return prevVal, nil
         }
         return fetch()
 }
 
-// doFetchNoPanic calls fetcher callback, trapping panics and validating
+// makeFetcherCtx prepares a context to use for fetch operation.
+//
+// Must be called under the lock.
+func (s *Slot) makeFetcherCtx(c context.Context) context.Context {
+        timeout := 15 * time.Second
+        if s.Timeout != 0 {
+                timeout = s.Timeout
+        }
+        fetcherCtx, _ := clock.WithTimeout(c, timeout)
+        return fetcherCtx
+}
+
+// fetchWithRetries calls fetchNoPanic, retrying on errors.
+func fetchWithRetries(ctx context.Context, cb Fetcher, factory retry.Factory, retryCb retry.Callback, prev Value) (result Value, err error) {
+        if factory == nil {
+                return fetchNoPanic(ctx, cb, prev)
+        }
+        err = retry.Retry(clock.Tag(ctx, "retry"), factory, func() error {
+                result, err = fetchNoPanic(ctx, cb, prev)
+                return err
+        }, retryCb)
+        if err != nil {
+                result = Value{}
+        }
+        return
+}
+
+// fetchNoPanic calls fetcher callback, trapping panics and validating
 // return value.
-func doFetchNoPanic(ctx context.Context, cb Fetcher, prev Value) (result Value, err error) {
+func fetchNoPanic(ctx context.Context, cb Fetcher, prev Value) (result Value, err error) {
         defer func() {
                 if r := recover(); r != nil {
                         err = fmt.Errorf("panic caught in lazyslot.Slot Fetcher - %s", r)
                         return
                 }
                 switch {
                 case err == nil && result.Value == nil:
                         err = fmt.Errorf("lazyslot.Slot Fetcher returned nil value")
                 case err != nil:
                         result = Value{}
                 }
         }()
         return cb(ctx, prev)
 }