pRPC: Retry on certain transient HTTP failures.

grpc/prpc/client_test.go

 // Copyright 2016 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package prpc
 
 import (
         "fmt"
         "io/ioutil"
         "net/http"
@@ skipping 50 matching lines @@
                 w.Header().Set("Content-Type", ContentTypePRPC)
                 _, err = w.Write(buf)
                 c.So(err, ShouldBeNil)
         }
 }
 
 func doPanicHandler(w http.ResponseWriter, r *http.Request) {
         panic("test panic")
 }
 
-func transientErrors(count int, then http.Handler) http.HandlerFunc {
+func transientErrors(count int, grpcHeader bool, then http.Handler) http.HandlerFunc {
         return func(w http.ResponseWriter, r *http.Request) {
                 if count > 0 {
                         count--
-                        w.Header().Set(HeaderGRPCCode, strconv.Itoa(int(codes.Internal)))
+                        if grpcHeader {
+                                w.Header().Set(HeaderGRPCCode, strconv.Itoa(int(codes.Internal)))
+                        }
                         w.WriteHeader(http.StatusInternalServerError)
                         fmt.Fprintln(w, "Server misbehaved")
                         return
                 }
                 then.ServeHTTP(w, r)
         }
 }
 
 func shouldHaveMessagesLike(actual interface{}, expected ...interface{}) string {
         log := actual.(*memlogger.MemLogger)
@@ skipping 90 matching lines @@
                         Convey(`When the response returns a huge Content Length, returns "ErrResponseTooBig".`, func(c C) {
                                 client, server := setUp(sayHello(c))
                                 defer server.Close()
 
                                 req.Name = "TOO BIG"
                                 err := client.Call(ctx, "prpc.Greeter", "SayHello", req, res)
                                 So(err, ShouldEqual, ErrResponseTooBig)
                         })
 
                         Convey("HTTP 500 x2", func(c C) {
-                                client, server := setUp(transientErrors(2, sayHello(c)))
+                                client, server := setUp(transientErrors(2, true, sayHello(c)))
                                 defer server.Close()
 
                                 err := client.Call(ctx, "prpc.Greeter", "SayHello", req, res)
                                 So(err, ShouldBeNil)
                                 So(res.Message, ShouldEqual, "Hello John")
 
                                 So(log, shouldHaveMessagesLike,
                                         expectedCallLogEntry(client),
                                         memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed transiently. Will retry in 0"},
 
                                         expectedCallLogEntry(client),
                                         memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed transiently. Will retry in 0"},
 
                                         expectedCallLogEntry(client),
                                 )
                         })
 
                         Convey("HTTP 500 many", func(c C) {
-                                client, server := setUp(transientErrors(10, sayHello(c)))
+                                client, server := setUp(transientErrors(10, true, sayHello(c)))
                                 defer server.Close()
 
                                 err := client.Call(ctx, "prpc.Greeter", "SayHello", req, res)
                                 So(grpc.Code(err), ShouldEqual, codes.Internal)
                                 So(grpc.ErrorDesc(err), ShouldEqual, "Server misbehaved")
 
                                 So(log, shouldHaveMessagesLike,
                                         expectedCallLogEntry(client),
                                         memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed transiently. Will retry in 0"},
 
                                         expectedCallLogEntry(client),
                                         memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed transiently. Will retry in 0"},
 
                                         expectedCallLogEntry(client),
                                         memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed transiently. Will retry in 0"},
 
                                         expectedCallLogEntry(client),
                                         memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed permanently"},
                                 )
                         })
 
+                        Convey("HTTP 500 without gRPC header", func(c C) {
+                                client, server := setUp(transientErrors(10, false, sayHello(c)))
+                                defer server.Close()
+
+                                err := client.Call(ctx, "prpc.Greeter", "SayHello", req, res)
+                                So(grpc.Code(err), ShouldEqual, codes.Unknown)
+
+                                So(log, shouldHaveMessagesLike,
+                                        expectedCallLogEntry(client),
+                                        memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed transiently. Will retry in 0"},
+
+                                        expectedCallLogEntry(client),
+                                        memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed transiently. Will retry in 0"},
+
+                                        expectedCallLogEntry(client),
+                                        memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed transiently. Will retry in 0"},
+
+                                        expectedCallLogEntry(client),
+                                        memlogger.LogEntry{Level: logging.Warning, Msg: "RPC failed permanently"},
+                                )
+                        })
+
                         Convey("Forbidden", func(c C) {
                                 client, server := setUp(func(w http.ResponseWriter, r *http.Request) {
                                         w.Header().Set(HeaderGRPCCode, strconv.Itoa(int(codes.PermissionDenied)))
                                         w.WriteHeader(http.StatusForbidden)
                                         fmt.Fprintln(w, "Access denied")
                                 })
                                 defer server.Close()
 
                                 err := client.Call(ctx, "prpc.Greeter", "SayHello", req, res)
                                 So(grpc.Code(err), ShouldEqual, codes.PermissionDenied)
@@ skipping 11 matching lines @@
                                         w.WriteHeader(http.StatusBadRequest)
                                 })
                                 defer server.Close()
 
                                 err := client.Call(ctx, "prpc.Greeter", "SayHello", req, res)
                                 So(grpc.Code(err), ShouldEqual, codes.Canceled)
                         })
                 })
         })
 }