pRPC: Retry on certain transient HTTP failures.

grpc/prpc/client.go

 // Copyright 2016 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package prpc
 
 import (
         "bytes"
         "fmt"
         "io"
@@ skipping 211 matching lines @@
                         if codeHeader == "" {
                                 // Not a valid pRPC response.
                                 body := buf.String()
                                 bodySize := c.ErrBodySize
                                 if bodySize <= 0 {
                                         bodySize = 256
                                 }
                                 if len(body) > bodySize {
                                         body = body[:bodySize] + "..."
                                 }
-                                return fmt.Errorf("HTTP %d: no gRPC code. Body: %q", res.StatusCode, body)
+                                err := fmt.Errorf("HTTP %d: no gRPC code. Body: %q", res.StatusCode, body)
+
+                                // Some HTTP codes are returned directly by hosting platforms (e.g.,
+                                // AppEngine), and should be automatically retried even if a gRPC code
+                                // header is not supplied.
+                                switch res.StatusCode {
+                                case http.StatusInternalServerError, http.StatusServiceUnavailable:
+                                        err = errors.WrapTransient(err)
+                                }
+                                return err
                         }
 
                         codeInt, err := strconv.Atoi(codeHeader)
                         if err != nil {
                                 // Not a valid pRPC response.
                                 return fmt.Errorf("invalid grpc code %q: %s", codeHeader, err)
                         }
 
                         code := codes.Code(codeInt)
                         if code != codes.OK {
@@ skipping 80 matching lines @@
         if len(h) == 0 {
                 return nil
         }
 
         md := make(metadata.MD, len(h))
         for k, v := range h {
                 md[strings.ToLower(k)] = v
         }
         return md
 }