OLD | NEW |
1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
4 | 4 |
5 package lhttp | 5 package lhttp |
6 | 6 |
7 import ( | 7 import ( |
8 "bytes" | 8 "bytes" |
9 "encoding/json" | 9 "encoding/json" |
10 "fmt" | 10 "fmt" |
11 "io" | 11 "io" |
12 "io/ioutil" | 12 "io/ioutil" |
13 "net/http" | 13 "net/http" |
14 "strings" | 14 "strings" |
15 | 15 |
16 "golang.org/x/net/context" | 16 "golang.org/x/net/context" |
17 | 17 |
18 "github.com/luci/luci-go/common/errors" | 18 "github.com/luci/luci-go/common/errors" |
19 "github.com/luci/luci-go/common/logging" | 19 "github.com/luci/luci-go/common/logging" |
20 "github.com/luci/luci-go/common/retry" | 20 "github.com/luci/luci-go/common/retry" |
| 21 "github.com/luci/luci-go/common/retry/transient" |
21 ) | 22 ) |
22 | 23 |
23 const ( | 24 const ( |
24 jsonContentType = "application/json" | 25 jsonContentType = "application/json" |
25 jsonContentTypeForPOST = "application/json; charset=utf-8" | 26 jsonContentTypeForPOST = "application/json; charset=utf-8" |
26 ) | 27 ) |
27 | 28 |
28 // Handler is called once or multiple times for each HTTP request that is tried. | 29 // Handler is called once or multiple times for each HTTP request that is tried. |
29 type Handler func(*http.Response) error | 30 type Handler func(*http.Response) error |
30 | 31 |
(...skipping 19 matching lines...) Expand all Loading... |
50 // example if a TCP connection is terminated while receiving the content. | 51 // example if a TCP connection is terminated while receiving the content. |
51 // | 52 // |
52 // If rFn is nil, NewRequest will use a default exponential backoff strategy | 53 // If rFn is nil, NewRequest will use a default exponential backoff strategy |
53 // only for transient errors. | 54 // only for transient errors. |
54 // | 55 // |
55 // If errorHandler is nil, the default error handler will drain and close the | 56 // If errorHandler is nil, the default error handler will drain and close the |
56 // response body. | 57 // response body. |
57 func NewRequest(ctx context.Context, c *http.Client, rFn retry.Factory, rgen Req
uestGen, | 58 func NewRequest(ctx context.Context, c *http.Client, rFn retry.Factory, rgen Req
uestGen, |
58 handler Handler, errorHandler ErrorHandler) func() (int, error) { | 59 handler Handler, errorHandler ErrorHandler) func() (int, error) { |
59 if rFn == nil { | 60 if rFn == nil { |
60 » » rFn = retry.TransientOnly(retry.Default) | 61 » » rFn = transient.Only(retry.Default) |
61 } | 62 } |
62 if errorHandler == nil { | 63 if errorHandler == nil { |
63 errorHandler = func(resp *http.Response, err error) error { | 64 errorHandler = func(resp *http.Response, err error) error { |
64 if resp != nil { | 65 if resp != nil { |
65 // Drain and close the resp.Body. | 66 // Drain and close the resp.Body. |
66 io.Copy(ioutil.Discard, resp.Body) | 67 io.Copy(ioutil.Discard, resp.Body) |
67 resp.Body.Close() | 68 resp.Body.Close() |
68 } | 69 } |
69 return err | 70 return err |
70 } | 71 } |
71 } | 72 } |
72 | 73 |
73 return func() (int, error) { | 74 return func() (int, error) { |
74 status, attempts := 0, 0 | 75 status, attempts := 0, 0 |
75 err := retry.Retry(ctx, rFn, func() error { | 76 err := retry.Retry(ctx, rFn, func() error { |
76 attempts++ | 77 attempts++ |
77 req, err := rgen() | 78 req, err := rgen() |
78 if err != nil { | 79 if err != nil { |
79 return err | 80 return err |
80 } | 81 } |
81 | 82 |
82 resp, err := c.Do(req) | 83 resp, err := c.Do(req) |
83 if err != nil { | 84 if err != nil { |
84 // Retry every error. This is sad when you speci
fy an invalid hostname but | 85 // Retry every error. This is sad when you speci
fy an invalid hostname but |
85 // it's better than failing when DNS resolution
is flaky. | 86 // it's better than failing when DNS resolution
is flaky. |
86 » » » » return errorHandler(nil, errors.WrapTransient(er
r)) | 87 » » » » return errorHandler(nil, transient.Tag.Apply(err
)) |
87 } | 88 } |
88 status = resp.StatusCode | 89 status = resp.StatusCode |
89 | 90 |
90 switch { | 91 switch { |
91 case status == 408, status == 429, status >= 500: | 92 case status == 408, status == 429, status >= 500: |
92 // The HTTP status code means the request should
be retried. | 93 // The HTTP status code means the request should
be retried. |
93 » » » » err = errors.WrapTransient( | 94 » » » » err = errors.Reason("http request failed: %(text
)s (HTTP %(code)d)"). |
94 » » » » » fmt.Errorf("http request failed: %s (HTT
P %d)", http.StatusText(status), status)) | 95 » » » » » D("text", http.StatusText(status)).D("co
de", status).Tag(transient.Tag).Err() |
95 case status == 404 && strings.HasPrefix(req.URL.Path, "/
_ah/api/"): | 96 case status == 404 && strings.HasPrefix(req.URL.Path, "/
_ah/api/"): |
96 // Endpoints occasionally return 404 on valid re
quests! | 97 // Endpoints occasionally return 404 on valid re
quests! |
97 logging.Infof(ctx, "lhttp.Do() got a Cloud Endpo
ints 404: %#v", resp.Header) | 98 logging.Infof(ctx, "lhttp.Do() got a Cloud Endpo
ints 404: %#v", resp.Header) |
98 » » » » err = errors.WrapTransient( | 99 » » » » err = errors.Reason("http request failed (endpoi
nts): %(text)s (HTTP %(code)d)"). |
99 » » » » » fmt.Errorf("http request failed (endpoin
ts): %s (HTTP %d)", http.StatusText(status), status)) | 100 » » » » » D("text", http.StatusText(status)).D("co
de", status).Tag(transient.Tag).Err() |
100 case status >= 400: | 101 case status >= 400: |
101 // Any other failure code is a hard failure. | 102 // Any other failure code is a hard failure. |
102 err = fmt.Errorf("http request failed: %s (HTTP
%d)", http.StatusText(status), status) | 103 err = fmt.Errorf("http request failed: %s (HTTP
%d)", http.StatusText(status), status) |
103 default: | 104 default: |
104 // The handler may still return a retry.Error to
indicate that the request | 105 // The handler may still return a retry.Error to
indicate that the request |
105 // should be retried even on successful status c
ode. | 106 // should be retried even on successful status c
ode. |
106 return handler(resp) | 107 return handler(resp) |
107 } | 108 } |
108 | 109 |
109 return errorHandler(resp, err) | 110 return errorHandler(resp, err) |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
150 // Non-retriable. | 151 // Non-retriable. |
151 return fmt.Errorf("unexpected Content-Type, expected \"%
s\", got \"%s\"", jsonContentType, ct) | 152 return fmt.Errorf("unexpected Content-Type, expected \"%
s\", got \"%s\"", jsonContentType, ct) |
152 } | 153 } |
153 if out == nil { | 154 if out == nil { |
154 // The client doesn't care about the response. Still ens
ure the response | 155 // The client doesn't care about the response. Still ens
ure the response |
155 // is valid json. | 156 // is valid json. |
156 out = &map[string]interface{}{} | 157 out = &map[string]interface{}{} |
157 } | 158 } |
158 if err := json.NewDecoder(resp.Body).Decode(out); err != nil { | 159 if err := json.NewDecoder(resp.Body).Decode(out); err != nil { |
159 // Retriable. | 160 // Retriable. |
160 » » » return errors.WrapTransient(fmt.Errorf("bad response %s:
%s", url, err)) | 161 » » » return errors.Annotate(err).Reason("bad response %(url)s
"). |
| 162 » » » » D("url", url).Tag(transient.Tag).Err() |
161 } | 163 } |
162 return nil | 164 return nil |
163 }, nil), nil | 165 }, nil), nil |
164 } | 166 } |
165 | 167 |
166 // GetJSON is a shorthand. It returns the HTTP status code and error if any. | 168 // GetJSON is a shorthand. It returns the HTTP status code and error if any. |
167 func GetJSON(ctx context.Context, rFn retry.Factory, c *http.Client, url string,
out interface{}) (int, error) { | 169 func GetJSON(ctx context.Context, rFn retry.Factory, c *http.Client, url string,
out interface{}) (int, error) { |
168 req, err := NewRequestJSON(ctx, c, rFn, url, "GET", nil, nil, out) | 170 req, err := NewRequestJSON(ctx, c, rFn, url, "GET", nil, nil, out) |
169 if err != nil { | 171 if err != nil { |
170 return 0, err | 172 return 0, err |
171 } | 173 } |
172 return req() | 174 return req() |
173 } | 175 } |
174 | 176 |
175 // PostJSON is a shorthand. It returns the HTTP status code and error if any. | 177 // PostJSON is a shorthand. It returns the HTTP status code and error if any. |
176 func PostJSON(ctx context.Context, rFn retry.Factory, c *http.Client, url string
, headers map[string]string, in, out interface{}) (int, error) { | 178 func PostJSON(ctx context.Context, rFn retry.Factory, c *http.Client, url string
, headers map[string]string, in, out interface{}) (int, error) { |
177 req, err := NewRequestJSON(ctx, c, rFn, url, "POST", headers, in, out) | 179 req, err := NewRequestJSON(ctx, c, rFn, url, "POST", headers, in, out) |
178 if err != nil { | 180 if err != nil { |
179 return 0, err | 181 return 0, err |
180 } | 182 } |
181 return req() | 183 return req() |
182 } | 184 } |
OLD | NEW |