Retry RPC deadlines when fetching GAE settings.

common/lazyslot/lazyslot_test.go

Index: common/lazyslot/lazyslot_test.go
diff --git a/common/lazyslot/lazyslot_test.go b/common/lazyslot/lazyslot_test.go
index e46f69f25921b7b95ee934d604054a4d7762baa8..483d377af53adc429313e090d5eaebf86eca8cbb 100644
--- a/common/lazyslot/lazyslot_test.go
+++ b/common/lazyslot/lazyslot_test.go
@@ -5,14 +5,18 @@
 package lazyslot
 
 import (
+	"fmt"
 	"sync"
 	"testing"
 	"time"
 
 	"golang.org/x/net/context"
 
+	"github.com/luci/luci-go/common/clock"
 	"github.com/luci/luci-go/common/clock/testclock"
+	"github.com/luci/luci-go/common/retry"
 
+	. "github.com/luci/luci-go/common/testing/assertions"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
@@ -137,6 +141,60 @@ func TestLazySlot(t *testing.T) {
 		}
 		So(func() { s.Get(c) }, ShouldPanicWith, "lazyslot.Slot Fetcher returned nil value")
 	})
+
+	Convey("Does retries", t, func(conv C) {
+		c, clk := newContext()
+
+		clk.SetTimerCallback(func(d time.Duration, t clock.Timer) {
+			if testclock.HasTags(t, "retry") {
+				clk.Add(d)
+			}
+		})
+
+		lock := sync.Mutex{}
+		calls := 0
+
+		s := Slot{
+			RetryFactory: retry.Default,
+			Fetcher: func(c context.Context, prev Value) (Value, error) {
+				lock.Lock()
+				defer lock.Unlock()
+				calls++
+				return Value{}, fmt.Errorf("omg, error")
+			},
+		}
+		_, err := s.Get(c)
+		So(err, ShouldErrLike, "context deadline exceeded")
+		So(calls, ShouldEqual, 8) // depends on retry config and deadline
+	})
+
+	Convey("Panics in retries", t, func(conv C) {
+		c, clk := newContext()
+
+		clk.SetTimerCallback(func(d time.Duration, t clock.Timer) {
+			if testclock.HasTags(t, "retry") {
+				clk.Add(d)
+			}
+		})
+
+		lock := sync.Mutex{}
+		calls := 0
+
+		s := Slot{
+			RetryFactory: retry.Default,
+			Fetcher: func(c context.Context, prev Value) (Value, error) {
+				lock.Lock()
+				defer lock.Unlock()
+				calls++
+				if calls == 4 {
+					panic("omg, panic")
+				}
+				return Value{}, fmt.Errorf("omg, error")
+			},
+		}
+		So(func() { s.Get(c) }, ShouldPanicWith, "omg, panic")
+		So(calls, ShouldEqual, 4)
+	})
 }
 
 func newContext() (context.Context, testclock.TestClock) {