OLD | NEW |
| (Empty) |
1 # Copyright 2015 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 import collections | |
6 import copy | |
7 import json | |
8 import logging | |
9 import os | |
10 import re | |
11 import socket | |
12 import sys | |
13 import time | |
14 | |
15 import httplib2 | |
16 import oauth2client.client | |
17 | |
18 from googleapiclient import errors | |
19 from infra_libs.ts_mon.common import http_metrics | |
20 | |
21 DEFAULT_SCOPES = ['email'] | |
22 | |
23 # default timeout for http requests, in seconds | |
24 DEFAULT_TIMEOUT = 30 | |
25 | |
26 # This is part of the API. | |
27 if sys.platform.startswith('win'): # pragma: no cover | |
28 SERVICE_ACCOUNTS_CREDS_ROOT = 'C:\\creds\\service_accounts' | |
29 else: | |
30 SERVICE_ACCOUNTS_CREDS_ROOT = '/creds/service_accounts' | |
31 | |
32 | |
33 class AuthError(Exception): | |
34 pass | |
35 | |
36 | |
37 def load_service_account_credentials(credentials_filename, | |
38 service_accounts_creds_root=None): | |
39 """Loads and validate a credential JSON file. | |
40 | |
41 Example of a well-formatted file: | |
42 { | |
43 "private_key_id": "4168d274cdc7a1eaef1c59f5b34bdf255", | |
44 "private_key": ("-----BEGIN PRIVATE KEY-----\nMIIhkiG9w0BAQEFAASCAmEwsd" | |
45 "sdfsfFd\ngfxFChctlOdTNm2Wrr919Nx9q+sPV5ibyaQt5Dgn89fKV" | |
46 "jftrO3AMDS3sMjaE4Ib\nZwJgy90wwBbMT7/YOzCgf5PZfivUe8KkB" | |
47 -----END PRIVATE KEY-----\n", | |
48 "client_email": "234243-rjstu8hi95iglc8at3@developer.gserviceaccount.com", | |
49 "client_id": "234243-rjstu8hi95iglc8at3.apps.googleusercontent.com", | |
50 "type": "service_account" | |
51 } | |
52 | |
53 Args: | |
54 credentials_filename (str): path to a .json file containing credentials | |
55 for a Cloud platform service account. | |
56 | |
57 Keyword Args: | |
58 service_accounts_creds_root (str or None): location where all service | |
59 account credentials are stored. ``credentials_filename`` is relative | |
60 to this path. None means 'use default location'. | |
61 | |
62 Raises: | |
63 AuthError: if the file content is invalid. | |
64 """ | |
65 service_accounts_creds_root = (service_accounts_creds_root | |
66 or SERVICE_ACCOUNTS_CREDS_ROOT) | |
67 | |
68 service_account_file = os.path.join(service_accounts_creds_root, | |
69 credentials_filename) | |
70 try: | |
71 with open(service_account_file, 'r') as f: | |
72 key = json.load(f) | |
73 except ValueError as e: | |
74 raise AuthError('Parsing of file as JSON failed (%s): %s', | |
75 e, service_account_file) | |
76 | |
77 if key.get('type') != 'service_account': | |
78 msg = ('Credentials type must be for a service_account, got %s.' | |
79 ' Check content of %s' % (key.get('type'), service_account_file)) | |
80 logging.error(msg) | |
81 raise AuthError(msg) | |
82 | |
83 if not key.get('client_email'): | |
84 msg = ('client_email field missing in credentials json file. ' | |
85 ' Check content of %s' % service_account_file) | |
86 logging.error(msg) | |
87 raise AuthError(msg) | |
88 | |
89 if not key.get('private_key'): | |
90 msg = ('private_key field missing in credentials json. ' | |
91 ' Check content of %s' % service_account_file) | |
92 logging.error(msg) | |
93 raise AuthError(msg) | |
94 | |
95 return key | |
96 | |
97 | |
98 def get_signed_jwt_assertion_credentials(credentials_filename, | |
99 scope=None, | |
100 service_accounts_creds_root=None): | |
101 """Factory for SignedJwtAssertionCredentials | |
102 | |
103 Reads and validate the json credential file. | |
104 | |
105 Args: | |
106 credentials_filename (str): path to the service account key file. | |
107 See load_service_account_credentials() docstring for the file format. | |
108 | |
109 Keyword Args: | |
110 scope (str|list of str): scope(s) of the credentials being | |
111 requested. Defaults to https://www.googleapis.com/auth/userinfo.email. | |
112 service_accounts_creds_root (str or None): location where all service | |
113 account credentials are stored. ``credentials_filename`` is relative | |
114 to this path. None means 'use default location'. | |
115 """ | |
116 scope = scope or DEFAULT_SCOPES | |
117 if isinstance(scope, basestring): | |
118 scope = [scope] | |
119 assert all(isinstance(s, basestring) for s in scope) | |
120 | |
121 key = load_service_account_credentials( | |
122 credentials_filename, | |
123 service_accounts_creds_root=service_accounts_creds_root) | |
124 | |
125 return oauth2client.client.SignedJwtAssertionCredentials( | |
126 key['client_email'], key['private_key'], scope) | |
127 | |
128 | |
129 def get_authenticated_http(credentials_filename, | |
130 scope=None, | |
131 service_accounts_creds_root=None, | |
132 http_identifier=None, | |
133 timeout=DEFAULT_TIMEOUT): | |
134 """Creates an httplib2.Http wrapped with a service account authenticator. | |
135 | |
136 Args: | |
137 credentials_filename (str): relative path to the file containing | |
138 credentials in json format. Path is relative to the default | |
139 location where credentials are stored (platform-dependent). | |
140 | |
141 Keyword Args: | |
142 scope (str|list of str): scope(s) of the credentials being | |
143 requested. Defaults to https://www.googleapis.com/auth/userinfo.email. | |
144 service_accounts_creds_root (str or None): location where all service | |
145 account credentials are stored. ``credentials_filename`` is relative | |
146 to this path. None means 'use default location'. | |
147 http_identifier (str): if provided, returns an instrumented http request | |
148 and use this string to identify it to ts_mon. | |
149 timeout (int): timeout passed to httplib2.Http, in seconds. | |
150 | |
151 Returns: | |
152 httplib2.Http authenticated with master's service account. | |
153 """ | |
154 creds = get_signed_jwt_assertion_credentials( | |
155 credentials_filename, | |
156 scope=scope, | |
157 service_accounts_creds_root=service_accounts_creds_root) | |
158 | |
159 if http_identifier: | |
160 http = InstrumentedHttp(http_identifier, timeout=timeout) | |
161 else: | |
162 http = httplib2.Http(timeout=timeout) | |
163 return creds.authorize(http) | |
164 | |
165 class RetriableHttp(object): | |
166 """A httplib2.Http object that retries on failure.""" | |
167 | |
168 def __init__(self, http, max_tries=5, backoff_time=1, | |
169 retrying_statuses_fn=None): | |
170 """ | |
171 Args: | |
172 http: an httplib2.Http instance | |
173 max_tries: a number of maximum tries | |
174 backoff_time: a number of seconds to sleep between retries | |
175 retrying_statuses_fn: a function that returns True if a given status | |
176 should be retried | |
177 """ | |
178 self._http = http | |
179 self._max_tries = max_tries | |
180 self._backoff_time = backoff_time | |
181 self._retrying_statuses_fn = retrying_statuses_fn or \ | |
182 set(range(500,599)).__contains__ | |
183 | |
184 def request(self, uri, method='GET', body=None, *args, **kwargs): | |
185 for i in range(1, self._max_tries + 1): | |
186 try: | |
187 response, content = self._http.request(uri, method, body, *args, | |
188 **kwargs) | |
189 | |
190 if self._retrying_statuses_fn(response.status): | |
191 logging.info('RetriableHttp: attempt %d receiving status %d, %s', | |
192 i, response.status, | |
193 'final attempt' if i == self._max_tries else \ | |
194 'will retry') | |
195 else: | |
196 break | |
197 except (ValueError, errors.Error, | |
198 socket.timeout, socket.error, socket.herror, socket.gaierror, | |
199 httplib2.HttpLib2Error) as error: | |
200 logging.info('RetriableHttp: attempt %d received exception: %s, %s', | |
201 i, error, 'final attempt' if i == self._max_tries else \ | |
202 'will retry') | |
203 if i == self._max_tries: | |
204 raise | |
205 time.sleep(self._backoff_time) | |
206 | |
207 return response, content | |
208 | |
209 def __getattr__(self, name): | |
210 return getattr(self._http, name) | |
211 | |
212 def __setattr__(self, name, value): | |
213 if name in ('request', '_http', '_max_tries', '_backoff_time', | |
214 '_retrying_statuses_fn'): | |
215 self.__dict__[name] = value | |
216 else: | |
217 setattr(self._http, name, value) | |
218 | |
219 class InstrumentedHttp(httplib2.Http): | |
220 """A httplib2.Http object that reports ts_mon metrics about its requests.""" | |
221 | |
222 def __init__(self, name, time_fn=time.time, timeout=DEFAULT_TIMEOUT, | |
223 **kwargs): | |
224 """ | |
225 Args: | |
226 name: An identifier for the HTTP requests made by this object. | |
227 time_fn: Function returning the current time in seconds. Use for testing | |
228 purposes only. | |
229 """ | |
230 | |
231 super(InstrumentedHttp, self).__init__(timeout=timeout, **kwargs) | |
232 self.fields = {'name': name, 'client': 'httplib2'} | |
233 self.time_fn = time_fn | |
234 | |
235 def _update_metrics(self, status, start_time): | |
236 status_fields = {'status': status} | |
237 status_fields.update(self.fields) | |
238 http_metrics.response_status.increment(fields=status_fields) | |
239 | |
240 duration_msec = (self.time_fn() - start_time) * 1000 | |
241 http_metrics.durations.add(duration_msec, fields=self.fields) | |
242 | |
243 def request(self, uri, method="GET", body=None, *args, **kwargs): | |
244 request_bytes = 0 | |
245 if body is not None: | |
246 request_bytes = len(body) | |
247 http_metrics.request_bytes.add(request_bytes, fields=self.fields) | |
248 | |
249 start_time = self.time_fn() | |
250 try: | |
251 response, content = super(InstrumentedHttp, self).request( | |
252 uri, method, body, *args, **kwargs) | |
253 except socket.timeout: | |
254 self._update_metrics(http_metrics.STATUS_TIMEOUT, start_time) | |
255 raise | |
256 except (socket.error, socket.herror, socket.gaierror): | |
257 self._update_metrics(http_metrics.STATUS_ERROR, start_time) | |
258 raise | |
259 except httplib2.HttpLib2Error: | |
260 self._update_metrics(http_metrics.STATUS_EXCEPTION, start_time) | |
261 raise | |
262 http_metrics.response_bytes.add(len(content), fields=self.fields) | |
263 | |
264 self._update_metrics(response.status, start_time) | |
265 | |
266 return response, content | |
267 | |
268 | |
269 class HttpMock(object): | |
270 """Mock of httplib2.Http""" | |
271 HttpCall = collections.namedtuple('HttpCall', ('uri', 'method', 'body', | |
272 'headers')) | |
273 | |
274 def __init__(self, uris): | |
275 """ | |
276 Args: | |
277 uris(dict): list of (uri, headers, body). `uri` is a regexp for | |
278 matching the requested uri, (headers, body) gives the values returned | |
279 by the mock. Uris are tested in the order from `uris`. | |
280 `headers` is a dict mapping headers to value. The 'status' key is | |
281 mandatory. `body` is a string. | |
282 Ex: [('.*', {'status': 200}, 'nicely done.')] | |
283 """ | |
284 self._uris = [] | |
285 self.requests_made = [] | |
286 | |
287 for value in uris: | |
288 if not isinstance(value, (list, tuple)) or len(value) != 3: | |
289 raise ValueError("'uris' must be a sequence of (uri, headers, body)") | |
290 uri, headers, body = value | |
291 compiled_uri = re.compile(uri) | |
292 if not isinstance(headers, dict): | |
293 raise TypeError("'headers' must be a dict") | |
294 if not 'status' in headers: | |
295 raise ValueError("'headers' must have 'status' as a key") | |
296 | |
297 new_headers = copy.copy(headers) | |
298 new_headers['status'] = int(new_headers['status']) | |
299 | |
300 if not isinstance(body, basestring): | |
301 raise TypeError("'body' must be a string, got %s" % type(body)) | |
302 self._uris.append((compiled_uri, new_headers, body)) | |
303 | |
304 # pylint: disable=unused-argument | |
305 def request(self, uri, | |
306 method='GET', | |
307 body=None, | |
308 headers=None, | |
309 redirections=1, | |
310 connection_type=None): | |
311 self.requests_made.append(self.HttpCall(uri, method, body, headers)) | |
312 headers = None | |
313 body = None | |
314 for candidate in self._uris: | |
315 if candidate[0].match(uri): | |
316 _, headers, body = candidate | |
317 break | |
318 if not headers: | |
319 raise AssertionError("Unexpected request to %s" % uri) | |
320 return httplib2.Response(headers), body | |
OLD | NEW |