OLD | NEW |
1 # Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ | 1 # Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ |
2 # Copyright (c) 2010 Google | 2 # Copyright (c) 2010 Google |
3 # Copyright (c) 2008 rPath, Inc. | 3 # Copyright (c) 2008 rPath, Inc. |
4 # Copyright (c) 2009 The Echo Nest Corporation | 4 # Copyright (c) 2009 The Echo Nest Corporation |
5 # Copyright (c) 2010, Eucalyptus Systems, Inc. | 5 # Copyright (c) 2010, Eucalyptus Systems, Inc. |
| 6 # Copyright (c) 2011, Nexenta Systems Inc. |
6 # All rights reserved. | 7 # All rights reserved. |
7 # | 8 # |
8 # Permission is hereby granted, free of charge, to any person obtaining a | 9 # Permission is hereby granted, free of charge, to any person obtaining a |
9 # copy of this software and associated documentation files (the | 10 # copy of this software and associated documentation files (the |
10 # "Software"), to deal in the Software without restriction, including | 11 # "Software"), to deal in the Software without restriction, including |
11 # without limitation the rights to use, copy, modify, merge, publish, dis- | 12 # without limitation the rights to use, copy, modify, merge, publish, dis- |
12 # tribute, sublicense, and/or sell copies of the Software, and to permit | 13 # tribute, sublicense, and/or sell copies of the Software, and to permit |
13 # persons to whom the Software is furnished to do so, subject to the fol- | 14 # persons to whom the Software is furnished to do so, subject to the fol- |
14 # lowing conditions: | 15 # lowing conditions: |
15 # | 16 # |
(...skipping 18 matching lines...) Expand all Loading... |
34 # you do not remove any proprietary notices. Your use of this software | 35 # you do not remove any proprietary notices. Your use of this software |
35 # code is at your own risk and you waive any claim against Amazon | 36 # code is at your own risk and you waive any claim against Amazon |
36 # Digital Services, Inc. or its affiliates with respect to your use of | 37 # Digital Services, Inc. or its affiliates with respect to your use of |
37 # this software code. (c) 2006 Amazon Digital Services, Inc. or its | 38 # this software code. (c) 2006 Amazon Digital Services, Inc. or its |
38 # affiliates. | 39 # affiliates. |
39 | 40 |
40 """ | 41 """ |
41 Handles basic connections to AWS | 42 Handles basic connections to AWS |
42 """ | 43 """ |
43 | 44 |
| 45 from __future__ import with_statement |
44 import base64 | 46 import base64 |
45 import errno | 47 import errno |
46 import httplib | 48 import httplib |
47 import os | 49 import os |
48 import Queue | 50 import Queue |
| 51 import random |
49 import re | 52 import re |
50 import socket | 53 import socket |
51 import sys | 54 import sys |
52 import time | 55 import time |
53 import urllib, urlparse | 56 import urllib, urlparse |
54 import xml.sax | 57 import xml.sax |
55 | 58 |
56 import auth | 59 import auth |
57 import auth_handler | 60 import auth_handler |
58 import boto | 61 import boto |
59 import boto.utils | 62 import boto.utils |
60 | 63 import boto.handler |
61 from boto import config, UserAgent, handler | 64 import boto.cacerts |
| 65 |
| 66 from boto import config, UserAgent |
62 from boto.exception import AWSConnectionError, BotoClientError, BotoServerError | 67 from boto.exception import AWSConnectionError, BotoClientError, BotoServerError |
63 from boto.provider import Provider | 68 from boto.provider import Provider |
64 from boto.resultset import ResultSet | 69 from boto.resultset import ResultSet |
65 | 70 |
| 71 HAVE_HTTPS_CONNECTION = False |
| 72 try: |
| 73 import ssl |
| 74 from boto import https_connection |
| 75 # Google App Engine runs on Python 2.5 so doesn't have ssl.SSLError. |
| 76 if hasattr(ssl, 'SSLError'): |
| 77 HAVE_HTTPS_CONNECTION = True |
| 78 except ImportError: |
| 79 pass |
| 80 |
| 81 try: |
| 82 import threading |
| 83 except ImportError: |
| 84 import dummy_threading as threading |
| 85 |
| 86 ON_APP_ENGINE = all(key in os.environ for key in ( |
| 87 'USER_IS_ADMIN', 'CURRENT_VERSION_ID', 'APPLICATION_ID')) |
66 | 88 |
67 PORTS_BY_SECURITY = { True: 443, False: 80 } | 89 PORTS_BY_SECURITY = { True: 443, False: 80 } |
68 | 90 |
69 class ConnectionPool: | 91 DEFAULT_CA_CERTS_FILE = os.path.join( |
70 def __init__(self, hosts, connections_per_host): | 92 os.path.dirname(os.path.abspath(boto.cacerts.__file__ )), "cacerts.txt") |
71 self._hosts = boto.utils.LRUCache(hosts) | 93 |
72 self.connections_per_host = connections_per_host | 94 class HostConnectionPool(object): |
73 | 95 |
74 def __getitem__(self, key): | 96 """ |
75 if key not in self._hosts: | 97 A pool of connections for one remote (host,is_secure). |
76 self._hosts[key] = Queue.Queue(self.connections_per_host) | 98 |
77 return self._hosts[key] | 99 When connections are added to the pool, they are put into a |
78 | 100 pending queue. The _mexe method returns connections to the pool |
79 def __repr__(self): | 101 before the response body has been read, so they connections aren't |
80 return 'ConnectionPool:%s' % ','.join(self._hosts._dict.keys()) | 102 ready to send another request yet. They stay in the pending queue |
| 103 until they are ready for another request, at which point they are |
| 104 returned to the pool of ready connections. |
| 105 |
| 106 The pool of ready connections is an ordered list of |
| 107 (connection,time) pairs, where the time is the time the connection |
| 108 was returned from _mexe. After a certain period of time, |
| 109 connections are considered stale, and discarded rather than being |
| 110 reused. This saves having to wait for the connection to time out |
| 111 if AWS has decided to close it on the other end because of |
| 112 inactivity. |
| 113 |
| 114 Thread Safety: |
| 115 |
| 116 This class is used only fram ConnectionPool while it's mutex |
| 117 is held. |
| 118 """ |
| 119 |
| 120 def __init__(self): |
| 121 self.queue = [] |
| 122 |
| 123 def size(self): |
| 124 """ |
| 125 Returns the number of connections in the pool for this host. |
| 126 Some of the connections may still be in use, and may not be |
| 127 ready to be returned by get(). |
| 128 """ |
| 129 return len(self.queue) |
| 130 |
| 131 def put(self, conn): |
| 132 """ |
| 133 Adds a connection to the pool, along with the time it was |
| 134 added. |
| 135 """ |
| 136 self.queue.append((conn, time.time())) |
| 137 |
| 138 def get(self): |
| 139 """ |
| 140 Returns the next connection in this pool that is ready to be |
| 141 reused. Returns None of there aren't any. |
| 142 """ |
| 143 # Discard ready connections that are too old. |
| 144 self.clean() |
| 145 |
| 146 # Return the first connection that is ready, and remove it |
| 147 # from the queue. Connections that aren't ready are returned |
| 148 # to the end of the queue with an updated time, on the |
| 149 # assumption that somebody is actively reading the response. |
| 150 for _ in range(len(self.queue)): |
| 151 (conn, _) = self.queue.pop(0) |
| 152 if self._conn_ready(conn): |
| 153 return conn |
| 154 else: |
| 155 self.put(conn) |
| 156 return None |
| 157 |
| 158 def _conn_ready(self, conn): |
| 159 """ |
| 160 There is a nice state diagram at the top of httplib.py. It |
| 161 indicates that once the response headers have been read (which |
| 162 _mexe does before adding the connection to the pool), a |
| 163 response is attached to the connection, and it stays there |
| 164 until it's done reading. This isn't entirely true: even after |
| 165 the client is done reading, the response may be closed, but |
| 166 not removed from the connection yet. |
| 167 |
| 168 This is ugly, reading a private instance variable, but the |
| 169 state we care about isn't available in any public methods. |
| 170 """ |
| 171 if ON_APP_ENGINE: |
| 172 # Google App Engine implementation of HTTPConnection doesn't contain |
| 173 # _HTTPConnection__response attribute. Moreover, it's not possible |
| 174 # to determine if given connection is ready. Reusing connections |
| 175 # simply doesn't make sense with App Engine urlfetch service. |
| 176 return False |
| 177 else: |
| 178 response = conn._HTTPConnection__response |
| 179 return (response is None) or response.isclosed() |
| 180 |
| 181 def clean(self): |
| 182 """ |
| 183 Get rid of stale connections. |
| 184 """ |
| 185 # Note that we do not close the connection here -- somebody |
| 186 # may still be reading from it. |
| 187 while len(self.queue) > 0 and self._pair_stale(self.queue[0]): |
| 188 self.queue.pop(0) |
| 189 |
| 190 def _pair_stale(self, pair): |
| 191 """ |
| 192 Returns true of the (connection,time) pair is too old to be |
| 193 used. |
| 194 """ |
| 195 (_conn, return_time) = pair |
| 196 now = time.time() |
| 197 return return_time + ConnectionPool.STALE_DURATION < now |
| 198 |
| 199 class ConnectionPool(object): |
| 200 |
| 201 """ |
| 202 A connection pool that expires connections after a fixed period of |
| 203 time. This saves time spent waiting for a connection that AWS has |
| 204 timed out on the other end. |
| 205 |
| 206 This class is thread-safe. |
| 207 """ |
| 208 |
| 209 # |
| 210 # The amout of time between calls to clean. |
| 211 # |
| 212 |
| 213 CLEAN_INTERVAL = 5.0 |
| 214 |
| 215 # |
| 216 # How long before a connection becomes "stale" and won't be reused |
| 217 # again. The intention is that this time is less that the timeout |
| 218 # period that AWS uses, so we'll never try to reuse a connection |
| 219 # and find that AWS is timing it out. |
| 220 # |
| 221 # Experimentation in July 2011 shows that AWS starts timing things |
| 222 # out after three minutes. The 60 seconds here is conservative so |
| 223 # we should never hit that 3-minute timout. |
| 224 # |
| 225 |
| 226 STALE_DURATION = 60.0 |
| 227 |
| 228 def __init__(self): |
| 229 # Mapping from (host,is_secure) to HostConnectionPool. |
| 230 # If a pool becomes empty, it is removed. |
| 231 self.host_to_pool = {} |
| 232 # The last time the pool was cleaned. |
| 233 self.last_clean_time = 0.0 |
| 234 self.mutex = threading.Lock() |
| 235 |
| 236 def size(self): |
| 237 """ |
| 238 Returns the number of connections in the pool. |
| 239 """ |
| 240 return sum(pool.size() for pool in self.host_to_pool.values()) |
| 241 |
| 242 def get_http_connection(self, host, is_secure): |
| 243 """ |
| 244 Gets a connection from the pool for the named host. Returns |
| 245 None if there is no connection that can be reused. |
| 246 """ |
| 247 self.clean() |
| 248 with self.mutex: |
| 249 key = (host, is_secure) |
| 250 if key not in self.host_to_pool: |
| 251 return None |
| 252 return self.host_to_pool[key].get() |
| 253 |
| 254 def put_http_connection(self, host, is_secure, conn): |
| 255 """ |
| 256 Adds a connection to the pool of connections that can be |
| 257 reused for the named host. |
| 258 """ |
| 259 with self.mutex: |
| 260 key = (host, is_secure) |
| 261 if key not in self.host_to_pool: |
| 262 self.host_to_pool[key] = HostConnectionPool() |
| 263 self.host_to_pool[key].put(conn) |
| 264 |
| 265 def clean(self): |
| 266 """ |
| 267 Clean up the stale connections in all of the pools, and then |
| 268 get rid of empty pools. Pools clean themselves every time a |
| 269 connection is fetched; this cleaning takes care of pools that |
| 270 aren't being used any more, so nothing is being gotten from |
| 271 them. |
| 272 """ |
| 273 with self.mutex: |
| 274 now = time.time() |
| 275 if self.last_clean_time + self.CLEAN_INTERVAL < now: |
| 276 to_remove = [] |
| 277 for (host, pool) in self.host_to_pool.items(): |
| 278 pool.clean() |
| 279 if pool.size() == 0: |
| 280 to_remove.append(host) |
| 281 for host in to_remove: |
| 282 del self.host_to_pool[host] |
| 283 self.last_clean_time = now |
81 | 284 |
82 class HTTPRequest(object): | 285 class HTTPRequest(object): |
83 | 286 |
84 def __init__(self, method, protocol, host, port, path, auth_path, | 287 def __init__(self, method, protocol, host, port, path, auth_path, |
85 params, headers, body): | 288 params, headers, body): |
86 """Represents an HTTP request. | 289 """Represents an HTTP request. |
87 | 290 |
88 :type method: string | 291 :type method: string |
89 :param method: The HTTP method name, 'GET', 'POST', 'PUT' etc. | 292 :param method: The HTTP method name, 'GET', 'POST', 'PUT' etc. |
90 | 293 |
91 :type protocol: string | 294 :type protocol: string |
92 :param protocol: The http protocol used, 'http' or 'https'. | 295 :param protocol: The http protocol used, 'http' or 'https'. |
93 | 296 |
94 :type host: string | 297 :type host: string |
95 :param host: Host to which the request is addressed. eg. abc.com | 298 :param host: Host to which the request is addressed. eg. abc.com |
96 | 299 |
97 :type port: int | 300 :type port: int |
98 :param port: port on which the request is being sent. Zero means unset, | 301 :param port: port on which the request is being sent. Zero means unset, |
99 in which case default port will be chosen. | 302 in which case default port will be chosen. |
100 | 303 |
101 :type path: string | 304 :type path: string |
102 :param path: URL path that is bein accessed. | 305 :param path: URL path that is bein accessed. |
103 | 306 |
104 :type auth_path: string | 307 :type auth_path: string |
105 :param path: The part of the URL path used when creating the | 308 :param path: The part of the URL path used when creating the |
106 authentication string. | 309 authentication string. |
107 | 310 |
108 :type params: dict | 311 :type params: dict |
109 :param params: HTTP url query parameters, with key as name of the param, | 312 :param params: HTTP url query parameters, with key as name of the param, |
110 and value as value of param. | 313 and value as value of param. |
111 | 314 |
112 :type headers: dict | 315 :type headers: dict |
113 :param headers: HTTP headers, with key as name of the header and value | 316 :param headers: HTTP headers, with key as name of the header and value |
114 as value of header. | 317 as value of header. |
115 | 318 |
116 :type body: string | 319 :type body: string |
117 :param body: Body of the HTTP request. If not present, will be None or | 320 :param body: Body of the HTTP request. If not present, will be None or |
118 empty string (''). | 321 empty string (''). |
119 """ | 322 """ |
120 self.method = method | 323 self.method = method |
121 self.protocol = protocol | 324 self.protocol = protocol |
122 self.host = host | 325 self.host = host |
123 self.port = port | 326 self.port = port |
124 self.path = path | 327 self.path = path |
| 328 if auth_path is None: |
| 329 auth_path = path |
125 self.auth_path = auth_path | 330 self.auth_path = auth_path |
126 self.params = params | 331 self.params = params |
127 self.headers = headers | 332 # chunked Transfer-Encoding should act only on PUT request. |
| 333 if headers and 'Transfer-Encoding' in headers and \ |
| 334 headers['Transfer-Encoding'] == 'chunked' and \ |
| 335 self.method != 'PUT': |
| 336 self.headers = headers.copy() |
| 337 del self.headers['Transfer-Encoding'] |
| 338 else: |
| 339 self.headers = headers |
128 self.body = body | 340 self.body = body |
129 | 341 |
130 def __str__(self): | 342 def __str__(self): |
131 return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) ' | 343 return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) ' |
132 'params(%s) headers(%s) body(%s)') % (self.method, | 344 'params(%s) headers(%s) body(%s)') % (self.method, |
133 self.protocol, self.host, self.port, self.path, self.params, | 345 self.protocol, self.host, self.port, self.path, self.params, |
134 self.headers, self.body)) | 346 self.headers, self.body)) |
135 | 347 |
| 348 def authorize(self, connection, **kwargs): |
| 349 for key in self.headers: |
| 350 val = self.headers[key] |
| 351 if isinstance(val, unicode): |
| 352 self.headers[key] = urllib.quote_plus(val.encode('utf-8')) |
| 353 |
| 354 connection._auth_handler.add_auth(self, **kwargs) |
| 355 |
| 356 self.headers['User-Agent'] = UserAgent |
| 357 # I'm not sure if this is still needed, now that add_auth is |
| 358 # setting the content-length for POST requests. |
| 359 if not self.headers.has_key('Content-Length'): |
| 360 if not self.headers.has_key('Transfer-Encoding') or \ |
| 361 self.headers['Transfer-Encoding'] != 'chunked': |
| 362 self.headers['Content-Length'] = str(len(self.body)) |
| 363 |
136 class AWSAuthConnection(object): | 364 class AWSAuthConnection(object): |
137 def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, | 365 def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, |
138 is_secure=True, port=None, proxy=None, proxy_port=None, | 366 is_secure=True, port=None, proxy=None, proxy_port=None, |
139 proxy_user=None, proxy_pass=None, debug=0, | 367 proxy_user=None, proxy_pass=None, debug=0, |
140 https_connection_factory=None, path='/', provider='aws'): | 368 https_connection_factory=None, path='/', |
| 369 provider='aws', security_token=None): |
141 """ | 370 """ |
142 :type host: str | 371 :type host: str |
143 :param host: The host to make the connection to | 372 :param host: The host to make the connection to |
144 | 373 |
145 :keyword str aws_access_key_id: Your AWS Access Key ID (provided by | 374 :keyword str aws_access_key_id: Your AWS Access Key ID (provided by |
146 Amazon). If none is specified, the value in your | 375 Amazon). If none is specified, the value in your |
147 ``AWS_ACCESS_KEY_ID`` environmental variable is used. | 376 ``AWS_ACCESS_KEY_ID`` environmental variable is used. |
148 :keyword str aws_secret_access_key: Your AWS Secret Access Key | 377 :keyword str aws_secret_access_key: Your AWS Secret Access Key |
149 (provided by Amazon). If none is specified, the value in your | 378 (provided by Amazon). If none is specified, the value in your |
150 ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. | 379 ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. |
151 | 380 |
152 :type is_secure: boolean | 381 :type is_secure: boolean |
153 :param is_secure: Whether the connection is over SSL | 382 :param is_secure: Whether the connection is over SSL |
154 | 383 |
155 :type https_connection_factory: list or tuple | 384 :type https_connection_factory: list or tuple |
156 :param https_connection_factory: A pair of an HTTP connection | 385 :param https_connection_factory: A pair of an HTTP connection |
157 factory and the exceptions to catch. | 386 factory and the exceptions to catch. |
158 The factory should have a similar | 387 The factory should have a similar |
159 interface to L{httplib.HTTPSConnection}
. | 388 interface to L{httplib.HTTPSConnection}
. |
160 | 389 |
161 :param str proxy: Address/hostname for a proxy server | 390 :param str proxy: Address/hostname for a proxy server |
162 | 391 |
163 :type proxy_port: int | 392 :type proxy_port: int |
164 :param proxy_port: The port to use when connecting over a proxy | 393 :param proxy_port: The port to use when connecting over a proxy |
165 | 394 |
166 :type proxy_user: str | 395 :type proxy_user: str |
167 :param proxy_user: The username to connect with on the proxy | 396 :param proxy_user: The username to connect with on the proxy |
168 | 397 |
169 :type proxy_pass: str | 398 :type proxy_pass: str |
170 :param proxy_pass: The password to use when connection over a proxy. | 399 :param proxy_pass: The password to use when connection over a proxy. |
171 | 400 |
172 :type port: int | 401 :type port: int |
173 :param port: The port to use to connect | 402 :param port: The port to use to connect |
174 """ | 403 """ |
175 self.num_retries = 5 | 404 self.num_retries = 6 |
176 # Override passed-in is_secure setting if value was defined in config. | 405 # Override passed-in is_secure setting if value was defined in config. |
177 if config.has_option('Boto', 'is_secure'): | 406 if config.has_option('Boto', 'is_secure'): |
178 is_secure = config.getboolean('Boto', 'is_secure') | 407 is_secure = config.getboolean('Boto', 'is_secure') |
179 self.is_secure = is_secure | 408 self.is_secure = is_secure |
| 409 # Whether or not to validate server certificates. At some point in the |
| 410 # future, the default should be flipped to true. |
| 411 self.https_validate_certificates = config.getbool( |
| 412 'Boto', 'https_validate_certificates', False) |
| 413 if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: |
| 414 raise BotoClientError( |
| 415 "SSL server certificate validation is enabled in boto " |
| 416 "configuration, but Python dependencies required to " |
| 417 "support this feature are not available. Certificate " |
| 418 "validation is only supported when running under Python " |
| 419 "2.6 or later.") |
| 420 self.ca_certificates_file = config.get_value( |
| 421 'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) |
180 self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) | 422 self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) |
181 # define exceptions from httplib that we want to catch and retry | 423 # define exceptions from httplib that we want to catch and retry |
182 self.http_exceptions = (httplib.HTTPException, socket.error, | 424 self.http_exceptions = (httplib.HTTPException, socket.error, |
183 socket.gaierror) | 425 socket.gaierror) |
| 426 # define subclasses of the above that are not retryable. |
| 427 self.http_unretryable_exceptions = [] |
| 428 if HAVE_HTTPS_CONNECTION: |
| 429 self.http_unretryable_exceptions.append(ssl.SSLError) |
| 430 self.http_unretryable_exceptions.append( |
| 431 https_connection.InvalidCertificateException) |
| 432 |
184 # define values in socket exceptions we don't want to catch | 433 # define values in socket exceptions we don't want to catch |
185 self.socket_exception_values = (errno.EINTR,) | 434 self.socket_exception_values = (errno.EINTR,) |
186 if https_connection_factory is not None: | 435 if https_connection_factory is not None: |
187 self.https_connection_factory = https_connection_factory[0] | 436 self.https_connection_factory = https_connection_factory[0] |
188 self.http_exceptions += https_connection_factory[1] | 437 self.http_exceptions += https_connection_factory[1] |
189 else: | 438 else: |
190 self.https_connection_factory = None | 439 self.https_connection_factory = None |
191 if (is_secure): | 440 if (is_secure): |
192 self.protocol = 'https' | 441 self.protocol = 'https' |
193 else: | 442 else: |
194 self.protocol = 'http' | 443 self.protocol = 'http' |
195 self.host = host | 444 self.host = host |
196 self.path = path | 445 self.path = path |
197 if debug: | 446 if debug: |
198 self.debug = debug | 447 self.debug = debug |
199 else: | 448 else: |
200 self.debug = config.getint('Boto', 'debug', debug) | 449 self.debug = config.getint('Boto', 'debug', debug) |
201 if port: | 450 if port: |
202 self.port = port | 451 self.port = port |
203 else: | 452 else: |
204 self.port = PORTS_BY_SECURITY[is_secure] | 453 self.port = PORTS_BY_SECURITY[is_secure] |
205 | 454 |
| 455 # Timeout used to tell httplib how long to wait for socket timeouts. |
| 456 # Default is to leave timeout unchanged, which will in turn result in |
| 457 # the socket's default global timeout being used. To specify a |
| 458 # timeout, set http_socket_timeout in Boto config. Regardless, |
| 459 # timeouts will only be applied if Python is 2.6 or greater. |
| 460 self.http_connection_kwargs = {} |
| 461 if (sys.version_info[0], sys.version_info[1]) >= (2, 6): |
| 462 if config.has_option('Boto', 'http_socket_timeout'): |
| 463 timeout = config.getint('Boto', 'http_socket_timeout') |
| 464 self.http_connection_kwargs['timeout'] = timeout |
| 465 |
206 self.provider = Provider(provider, | 466 self.provider = Provider(provider, |
207 aws_access_key_id, | 467 aws_access_key_id, |
208 aws_secret_access_key) | 468 aws_secret_access_key, |
| 469 security_token) |
209 | 470 |
210 # allow config file to override default host | 471 # allow config file to override default host |
211 if self.provider.host: | 472 if self.provider.host: |
212 self.host = self.provider.host | 473 self.host = self.provider.host |
213 | 474 |
214 # cache up to 20 connections per host, up to 20 hosts | 475 self._pool = ConnectionPool() |
215 self._pool = ConnectionPool(20, 20) | |
216 self._connection = (self.server_name(), self.is_secure) | 476 self._connection = (self.server_name(), self.is_secure) |
217 self._last_rs = None | 477 self._last_rs = None |
218 self._auth_handler = auth.get_auth_handler( | 478 self._auth_handler = auth.get_auth_handler( |
219 host, config, self.provider, self._required_auth_capability()) | 479 host, config, self.provider, self._required_auth_capability()) |
220 | 480 |
221 def __repr__(self): | 481 def __repr__(self): |
222 return '%s:%s' % (self.__class__.__name__, self.host) | 482 return '%s:%s' % (self.__class__.__name__, self.host) |
223 | 483 |
224 def _required_auth_capability(self): | 484 def _required_auth_capability(self): |
225 return [] | 485 return [] |
226 | 486 |
227 def _cached_name(self, host, is_secure): | |
228 if host is None: | |
229 host = self.server_name() | |
230 cached_name = is_secure and 'https://' or 'http://' | |
231 cached_name += host | |
232 return cached_name | |
233 | |
234 def connection(self): | 487 def connection(self): |
235 return self.get_http_connection(*self._connection) | 488 return self.get_http_connection(*self._connection) |
236 connection = property(connection) | 489 connection = property(connection) |
237 | 490 |
238 def aws_access_key_id(self): | 491 def aws_access_key_id(self): |
239 return self.provider.access_key | 492 return self.provider.access_key |
240 aws_access_key_id = property(aws_access_key_id) | 493 aws_access_key_id = property(aws_access_key_id) |
241 gs_access_key_id = aws_access_key_id | 494 gs_access_key_id = aws_access_key_id |
242 access_key = aws_access_key_id | 495 access_key = aws_access_key_id |
243 | 496 |
(...skipping 30 matching lines...) Expand all Loading... |
274 if port == 80: | 527 if port == 80: |
275 signature_host = self.host | 528 signature_host = self.host |
276 else: | 529 else: |
277 # This unfortunate little hack can be attributed to | 530 # This unfortunate little hack can be attributed to |
278 # a difference in the 2.6 version of httplib. In old | 531 # a difference in the 2.6 version of httplib. In old |
279 # versions, it would append ":443" to the hostname sent | 532 # versions, it would append ":443" to the hostname sent |
280 # in the Host header and so we needed to make sure we | 533 # in the Host header and so we needed to make sure we |
281 # did the same when calculating the V2 signature. In 2.6 | 534 # did the same when calculating the V2 signature. In 2.6 |
282 # (and higher!) | 535 # (and higher!) |
283 # it no longer does that. Hence, this kludge. | 536 # it no longer does that. Hence, this kludge. |
284 if sys.version[:3] in ('2.6', '2.7') and port == 443: | 537 if ((ON_APP_ENGINE and sys.version[:3] == '2.5') or |
| 538 sys.version[:3] in ('2.6', '2.7')) and port == 443: |
285 signature_host = self.host | 539 signature_host = self.host |
286 else: | 540 else: |
287 signature_host = '%s:%d' % (self.host, port) | 541 signature_host = '%s:%d' % (self.host, port) |
288 return signature_host | 542 return signature_host |
289 | 543 |
290 def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): | 544 def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): |
291 self.proxy = proxy | 545 self.proxy = proxy |
292 self.proxy_port = proxy_port | 546 self.proxy_port = proxy_port |
293 self.proxy_user = proxy_user | 547 self.proxy_user = proxy_user |
294 self.proxy_pass = proxy_pass | 548 self.proxy_pass = proxy_pass |
(...skipping 20 matching lines...) Expand all Loading... |
315 if not self.proxy_pass: | 569 if not self.proxy_pass: |
316 self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) | 570 self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) |
317 | 571 |
318 if not self.proxy_port and self.proxy: | 572 if not self.proxy_port and self.proxy: |
319 print "http_proxy environment variable does not specify " \ | 573 print "http_proxy environment variable does not specify " \ |
320 "a port, using default" | 574 "a port, using default" |
321 self.proxy_port = self.port | 575 self.proxy_port = self.port |
322 self.use_proxy = (self.proxy != None) | 576 self.use_proxy = (self.proxy != None) |
323 | 577 |
324 def get_http_connection(self, host, is_secure): | 578 def get_http_connection(self, host, is_secure): |
325 queue = self._pool[self._cached_name(host, is_secure)] | 579 conn = self._pool.get_http_connection(host, is_secure) |
326 try: | 580 if conn is not None: |
327 return queue.get_nowait() | 581 return conn |
328 except Queue.Empty: | 582 else: |
329 return self.new_http_connection(host, is_secure) | 583 return self.new_http_connection(host, is_secure) |
330 | 584 |
331 def new_http_connection(self, host, is_secure): | 585 def new_http_connection(self, host, is_secure): |
332 if self.use_proxy: | 586 if self.use_proxy: |
333 host = '%s:%d' % (self.proxy, int(self.proxy_port)) | 587 host = '%s:%d' % (self.proxy, int(self.proxy_port)) |
334 if host is None: | 588 if host is None: |
335 host = self.server_name() | 589 host = self.server_name() |
336 if is_secure: | 590 if is_secure: |
337 boto.log.debug('establishing HTTPS connection') | 591 boto.log.debug( |
| 592 'establishing HTTPS connection: host=%s, kwargs=%s', |
| 593 host, self.http_connection_kwargs) |
338 if self.use_proxy: | 594 if self.use_proxy: |
339 connection = self.proxy_ssl() | 595 connection = self.proxy_ssl() |
340 elif self.https_connection_factory: | 596 elif self.https_connection_factory: |
341 connection = self.https_connection_factory(host) | 597 connection = self.https_connection_factory(host) |
| 598 elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION: |
| 599 connection = https_connection.CertValidatingHTTPSConnection( |
| 600 host, ca_certs=self.ca_certificates_file, |
| 601 **self.http_connection_kwargs) |
342 else: | 602 else: |
343 connection = httplib.HTTPSConnection(host) | 603 connection = httplib.HTTPSConnection(host, |
| 604 **self.http_connection_kwargs) |
344 else: | 605 else: |
345 boto.log.debug('establishing HTTP connection') | 606 boto.log.debug('establishing HTTP connection: kwargs=%s' % |
346 connection = httplib.HTTPConnection(host) | 607 self.http_connection_kwargs) |
| 608 connection = httplib.HTTPConnection(host, |
| 609 **self.http_connection_kwargs) |
347 if self.debug > 1: | 610 if self.debug > 1: |
348 connection.set_debuglevel(self.debug) | 611 connection.set_debuglevel(self.debug) |
349 # self.connection must be maintained for backwards-compatibility | 612 # self.connection must be maintained for backwards-compatibility |
350 # however, it must be dynamically pulled from the connection pool | 613 # however, it must be dynamically pulled from the connection pool |
351 # set a private variable which will enable that | 614 # set a private variable which will enable that |
352 if host.split(':')[0] == self.host and is_secure == self.is_secure: | 615 if host.split(':')[0] == self.host and is_secure == self.is_secure: |
353 self._connection = (host, is_secure) | 616 self._connection = (host, is_secure) |
354 return connection | 617 return connection |
355 | 618 |
356 def put_http_connection(self, host, is_secure, connection): | 619 def put_http_connection(self, host, is_secure, connection): |
357 try: | 620 self._pool.put_http_connection(host, is_secure, connection) |
358 self._pool[self._cached_name(host, is_secure)].put_nowait(connection
) | |
359 except Queue.Full: | |
360 # gracefully fail in case of pool overflow | |
361 connection.close() | |
362 | 621 |
363 def proxy_ssl(self): | 622 def proxy_ssl(self): |
364 host = '%s:%d' % (self.host, self.port) | 623 host = '%s:%d' % (self.host, self.port) |
365 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | 624 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
366 try: | 625 try: |
367 sock.connect((self.proxy, int(self.proxy_port))) | 626 sock.connect((self.proxy, int(self.proxy_port))) |
368 except: | 627 except: |
369 raise | 628 raise |
| 629 boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host) |
370 sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) | 630 sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) |
371 sock.sendall("User-Agent: %s\r\n" % UserAgent) | 631 sock.sendall("User-Agent: %s\r\n" % UserAgent) |
372 if self.proxy_user and self.proxy_pass: | 632 if self.proxy_user and self.proxy_pass: |
373 for k, v in self.get_proxy_auth_header().items(): | 633 for k, v in self.get_proxy_auth_header().items(): |
374 sock.sendall("%s: %s\r\n" % (k, v)) | 634 sock.sendall("%s: %s\r\n" % (k, v)) |
375 sock.sendall("\r\n") | 635 sock.sendall("\r\n") |
376 resp = httplib.HTTPResponse(sock, strict=True) | 636 resp = httplib.HTTPResponse(sock, strict=True, debuglevel=self.debug) |
377 resp.begin() | 637 resp.begin() |
378 | 638 |
379 if resp.status != 200: | 639 if resp.status != 200: |
380 # Fake a socket error, use a code that make it obvious it hasn't | 640 # Fake a socket error, use a code that make it obvious it hasn't |
381 # been generated by the socket library | 641 # been generated by the socket library |
382 raise socket.error(-71, | 642 raise socket.error(-71, |
383 "Error talking to HTTP proxy %s:%s: %s (%s)" % | 643 "Error talking to HTTP proxy %s:%s: %s (%s)" % |
384 (self.proxy, self.proxy_port, resp.status, resp.r
eason)) | 644 (self.proxy, self.proxy_port, resp.status, resp.r
eason)) |
385 | 645 |
386 # We can safely close the response, it duped the original socket | 646 # We can safely close the response, it duped the original socket |
387 resp.close() | 647 resp.close() |
388 | 648 |
389 h = httplib.HTTPConnection(host) | 649 h = httplib.HTTPConnection(host) |
390 | 650 |
391 # Wrap the socket in an SSL socket | 651 if self.https_validate_certificates and HAVE_HTTPS_CONNECTION: |
392 if hasattr(httplib, 'ssl'): | 652 boto.log.debug("wrapping ssl socket for proxied connection; " |
393 sslSock = httplib.ssl.SSLSocket(sock) | 653 "CA certificate file=%s", |
394 else: # Old Python, no ssl module | 654 self.ca_certificates_file) |
395 sslSock = socket.ssl(sock, None, None) | 655 key_file = self.http_connection_kwargs.get('key_file', None) |
396 sslSock = httplib.FakeSocket(sock, sslSock) | 656 cert_file = self.http_connection_kwargs.get('cert_file', None) |
| 657 sslSock = ssl.wrap_socket(sock, keyfile=key_file, |
| 658 certfile=cert_file, |
| 659 cert_reqs=ssl.CERT_REQUIRED, |
| 660 ca_certs=self.ca_certificates_file) |
| 661 cert = sslSock.getpeercert() |
| 662 hostname = self.host.split(':', 0)[0] |
| 663 if not https_connection.ValidateCertificateHostname(cert, hostname): |
| 664 raise https_connection.InvalidCertificateException( |
| 665 hostname, cert, 'hostname mismatch') |
| 666 else: |
| 667 # Fallback for old Python without ssl.wrap_socket |
| 668 if hasattr(httplib, 'ssl'): |
| 669 sslSock = httplib.ssl.SSLSocket(sock) |
| 670 else: |
| 671 sslSock = socket.ssl(sock, None, None) |
| 672 sslSock = httplib.FakeSocket(sock, sslSock) |
| 673 |
397 # This is a bit unclean | 674 # This is a bit unclean |
398 h.sock = sslSock | 675 h.sock = sslSock |
399 return h | 676 return h |
400 | 677 |
401 def prefix_proxy_to_path(self, path, host=None): | 678 def prefix_proxy_to_path(self, path, host=None): |
402 path = self.protocol + '://' + (host or self.server_name()) + path | 679 path = self.protocol + '://' + (host or self.server_name()) + path |
403 return path | 680 return path |
404 | 681 |
405 def get_proxy_auth_header(self): | 682 def get_proxy_auth_header(self): |
406 auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass) | 683 auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass) |
407 return {'Proxy-Authorization': 'Basic %s' % auth} | 684 return {'Proxy-Authorization': 'Basic %s' % auth} |
408 | 685 |
409 def _mexe(self, method, path, data, headers, host=None, sender=None, | 686 def _mexe(self, request, sender=None, override_num_retries=None): |
410 override_num_retries=None): | |
411 """ | 687 """ |
412 mexe - Multi-execute inside a loop, retrying multiple times to handle | 688 mexe - Multi-execute inside a loop, retrying multiple times to handle |
413 transient Internet errors by simply trying again. | 689 transient Internet errors by simply trying again. |
414 Also handles redirects. | 690 Also handles redirects. |
415 | 691 |
416 This code was inspired by the S3Utils classes posted to the boto-users | 692 This code was inspired by the S3Utils classes posted to the boto-users |
417 Google group by Larry Bates. Thanks! | 693 Google group by Larry Bates. Thanks! |
418 """ | 694 """ |
419 boto.log.debug('Method: %s' % method) | 695 boto.log.debug('Method: %s' % request.method) |
420 boto.log.debug('Path: %s' % path) | 696 boto.log.debug('Path: %s' % request.path) |
421 boto.log.debug('Data: %s' % data) | 697 boto.log.debug('Data: %s' % request.body) |
422 boto.log.debug('Headers: %s' % headers) | 698 boto.log.debug('Headers: %s' % request.headers) |
423 boto.log.debug('Host: %s' % host) | 699 boto.log.debug('Host: %s' % request.host) |
424 response = None | 700 response = None |
425 body = None | 701 body = None |
426 e = None | 702 e = None |
427 if override_num_retries is None: | 703 if override_num_retries is None: |
428 num_retries = config.getint('Boto', 'num_retries', self.num_retries) | 704 num_retries = config.getint('Boto', 'num_retries', self.num_retries) |
429 else: | 705 else: |
430 num_retries = override_num_retries | 706 num_retries = override_num_retries |
431 i = 0 | 707 i = 0 |
432 connection = self.get_http_connection(host, self.is_secure) | 708 connection = self.get_http_connection(request.host, self.is_secure) |
433 while i <= num_retries: | 709 while i <= num_retries: |
| 710 # Use binary exponential backoff to desynchronize client requests |
| 711 next_sleep = random.random() * (2 ** i) |
434 try: | 712 try: |
| 713 # we now re-sign each request before it is retried |
| 714 request.authorize(connection=self) |
435 if callable(sender): | 715 if callable(sender): |
436 response = sender(connection, method, path, data, headers) | 716 response = sender(connection, request.method, request.path, |
| 717 request.body, request.headers) |
437 else: | 718 else: |
438 connection.request(method, path, data, headers) | 719 connection.request(request.method, request.path, request.bod
y, |
| 720 request.headers) |
439 response = connection.getresponse() | 721 response = connection.getresponse() |
440 location = response.getheader('location') | 722 location = response.getheader('location') |
441 # -- gross hack -- | 723 # -- gross hack -- |
442 # httplib gets confused with chunked responses to HEAD requests | 724 # httplib gets confused with chunked responses to HEAD requests |
443 # so I have to fake it out | 725 # so I have to fake it out |
444 if method == 'HEAD' and getattr(response, 'chunked', False): | 726 if request.method == 'HEAD' and getattr(response, 'chunked', Fal
se): |
445 response.chunked = 0 | 727 response.chunked = 0 |
446 if response.status == 500 or response.status == 503: | 728 if response.status == 500 or response.status == 503: |
447 boto.log.debug('received %d response, retrying in %d seconds
' % (response.status, 2 ** i)) | 729 boto.log.debug('received %d response, retrying in %3.1f seco
nds' % |
| 730 (response.status, next_sleep)) |
448 body = response.read() | 731 body = response.read() |
449 elif response.status == 408: | |
450 body = response.read() | |
451 print '-------------------------' | |
452 print ' 4 0 8 ' | |
453 print 'path=%s' % path | |
454 print body | |
455 print '-------------------------' | |
456 elif response.status < 300 or response.status >= 400 or \ | 732 elif response.status < 300 or response.status >= 400 or \ |
457 not location: | 733 not location: |
458 self.put_http_connection(host, self.is_secure, connection) | 734 self.put_http_connection(request.host, self.is_secure, conne
ction) |
459 return response | 735 return response |
460 else: | 736 else: |
461 scheme, host, path, params, query, fragment = \ | 737 scheme, request.host, request.path, params, query, fragment
= \ |
462 urlparse.urlparse(location) | 738 urlparse.urlparse(location) |
463 if query: | 739 if query: |
464 path += '?' + query | 740 request.path += '?' + query |
465 boto.log.debug('Redirecting: %s' % scheme + '://' + host + p
ath) | 741 boto.log.debug('Redirecting: %s' % scheme + '://' + request.
host + request.path) |
466 connection = self.get_http_connection(host, scheme == 'https
') | 742 connection = self.get_http_connection(request.host, scheme =
= 'https') |
467 continue | 743 continue |
468 except KeyboardInterrupt: | |
469 sys.exit('Keyboard Interrupt') | |
470 except self.http_exceptions, e: | 744 except self.http_exceptions, e: |
| 745 for unretryable in self.http_unretryable_exceptions: |
| 746 if isinstance(e, unretryable): |
| 747 boto.log.debug( |
| 748 'encountered unretryable %s exception, re-raising' % |
| 749 e.__class__.__name__) |
| 750 raise e |
471 boto.log.debug('encountered %s exception, reconnecting' % \ | 751 boto.log.debug('encountered %s exception, reconnecting' % \ |
472 e.__class__.__name__) | 752 e.__class__.__name__) |
473 connection = self.new_http_connection(host, self.is_secure) | 753 connection = self.new_http_connection(request.host, self.is_secu
re) |
474 time.sleep(2 ** i) | 754 time.sleep(next_sleep) |
475 i += 1 | 755 i += 1 |
476 # If we made it here, it's because we have exhausted our retries and sti
l haven't | 756 # If we made it here, it's because we have exhausted our retries and sti
l haven't |
477 # succeeded. So, if we have a response object, use it to raise an excep
tion. | 757 # succeeded. So, if we have a response object, use it to raise an excep
tion. |
478 # Otherwise, raise the exception that must have already happened. | 758 # Otherwise, raise the exception that must have already happened. |
479 if response: | 759 if response: |
480 raise BotoServerError(response.status, response.reason, body) | 760 raise BotoServerError(response.status, response.reason, body) |
481 elif e: | 761 elif e: |
482 raise e | 762 raise e |
483 else: | 763 else: |
484 raise BotoClientError('Please report this exception as a Boto Issue!
') | 764 raise BotoClientError('Please report this exception as a Boto Issue!
') |
485 | 765 |
486 def build_base_http_request(self, method, path, auth_path, | 766 def build_base_http_request(self, method, path, auth_path, |
487 params=None, headers=None, data='', host=None): | 767 params=None, headers=None, data='', host=None): |
488 path = self.get_path(path) | 768 path = self.get_path(path) |
489 if auth_path is not None: | 769 if auth_path is not None: |
490 auth_path = self.get_path(auth_path) | 770 auth_path = self.get_path(auth_path) |
491 if params == None: | 771 if params == None: |
492 params = {} | 772 params = {} |
493 else: | 773 else: |
494 params = params.copy() | 774 params = params.copy() |
495 if headers == None: | 775 if headers == None: |
496 headers = {} | 776 headers = {} |
497 else: | 777 else: |
498 headers = headers.copy() | 778 headers = headers.copy() |
499 host = host or self.host | 779 host = host or self.host |
500 if self.use_proxy: | 780 if self.use_proxy: |
| 781 if not auth_path: |
| 782 auth_path = path |
501 path = self.prefix_proxy_to_path(path, host) | 783 path = self.prefix_proxy_to_path(path, host) |
502 if self.proxy_user and self.proxy_pass and not self.is_secure: | 784 if self.proxy_user and self.proxy_pass and not self.is_secure: |
503 # If is_secure, we don't have to set the proxy authentication | 785 # If is_secure, we don't have to set the proxy authentication |
504 # header here, we did that in the CONNECT to the proxy. | 786 # header here, we did that in the CONNECT to the proxy. |
505 headers.update(self.get_proxy_auth_header()) | 787 headers.update(self.get_proxy_auth_header()) |
506 return HTTPRequest(method, self.protocol, host, self.port, | 788 return HTTPRequest(method, self.protocol, host, self.port, |
507 path, auth_path, params, headers, data) | 789 path, auth_path, params, headers, data) |
508 | 790 |
509 def fill_in_auth(self, http_request, **kwargs): | |
510 headers = http_request.headers | |
511 for key in headers: | |
512 val = headers[key] | |
513 if isinstance(val, unicode): | |
514 headers[key] = urllib.quote_plus(val.encode('utf-8')) | |
515 | |
516 self._auth_handler.add_auth(http_request, **kwargs) | |
517 | |
518 headers['User-Agent'] = UserAgent | |
519 if not headers.has_key('Content-Length'): | |
520 headers['Content-Length'] = str(len(http_request.body)) | |
521 return http_request | |
522 | |
523 def _send_http_request(self, http_request, sender=None, | |
524 override_num_retries=None): | |
525 return self._mexe(http_request.method, http_request.path, | |
526 http_request.body, http_request.headers, | |
527 http_request.host, sender, override_num_retries) | |
528 | |
529 def make_request(self, method, path, headers=None, data='', host=None, | 791 def make_request(self, method, path, headers=None, data='', host=None, |
530 auth_path=None, sender=None, override_num_retries=None): | 792 auth_path=None, sender=None, override_num_retries=None): |
531 """Makes a request to the server, with stock multiple-retry logic.""" | 793 """Makes a request to the server, with stock multiple-retry logic.""" |
532 http_request = self.build_base_http_request(method, path, auth_path, | 794 http_request = self.build_base_http_request(method, path, auth_path, |
533 {}, headers, data, host) | 795 {}, headers, data, host) |
534 http_request = self.fill_in_auth(http_request) | 796 return self._mexe(http_request, sender, override_num_retries) |
535 return self._send_http_request(http_request, sender, | |
536 override_num_retries) | |
537 | 797 |
538 def close(self): | 798 def close(self): |
539 """(Optional) Close any open HTTP connections. This is non-destructive, | 799 """(Optional) Close any open HTTP connections. This is non-destructive, |
540 and making a new request will open a connection again.""" | 800 and making a new request will open a connection again.""" |
541 | 801 |
542 boto.log.debug('closing all HTTP connections') | 802 boto.log.debug('closing all HTTP connections') |
543 self.connection = None # compat field | 803 self.connection = None # compat field |
544 | 804 |
545 class AWSQueryConnection(AWSAuthConnection): | 805 class AWSQueryConnection(AWSAuthConnection): |
546 | 806 |
547 APIVersion = '' | 807 APIVersion = '' |
548 ResponseError = BotoServerError | 808 ResponseError = BotoServerError |
549 | 809 |
550 def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, | 810 def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, |
551 is_secure=True, port=None, proxy=None, proxy_port=None, | 811 is_secure=True, port=None, proxy=None, proxy_port=None, |
552 proxy_user=None, proxy_pass=None, host=None, debug=0, | 812 proxy_user=None, proxy_pass=None, host=None, debug=0, |
553 https_connection_factory=None, path='/'): | 813 https_connection_factory=None, path='/', security_token=None): |
554 AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_acc
ess_key, | 814 AWSAuthConnection.__init__(self, host, aws_access_key_id, |
555 is_secure, port, proxy, proxy_port, proxy_use
r, proxy_pass, | 815 aws_secret_access_key, |
556 debug, https_connection_factory, path) | 816 is_secure, port, proxy, |
| 817 proxy_port, proxy_user, proxy_pass, |
| 818 debug, https_connection_factory, path, |
| 819 security_token=security_token) |
557 | 820 |
558 def _required_auth_capability(self): | 821 def _required_auth_capability(self): |
559 return [] | 822 return [] |
560 | 823 |
561 def get_utf8_value(self, value): | 824 def get_utf8_value(self, value): |
562 return boto.utils.get_utf8_value(value) | 825 return boto.utils.get_utf8_value(value) |
563 | 826 |
564 def make_request(self, action, params=None, path='/', verb='GET'): | 827 def make_request(self, action, params=None, path='/', verb='GET'): |
565 http_request = self.build_base_http_request(verb, path, None, | 828 http_request = self.build_base_http_request(verb, path, None, |
566 params, {}, '', | 829 params, {}, '', |
567 self.server_name()) | 830 self.server_name()) |
568 if action: | 831 if action: |
569 http_request.params['Action'] = action | 832 http_request.params['Action'] = action |
570 http_request.params['Version'] = self.APIVersion | 833 http_request.params['Version'] = self.APIVersion |
571 http_request = self.fill_in_auth(http_request) | 834 return self._mexe(http_request) |
572 return self._send_http_request(http_request) | |
573 | 835 |
574 def build_list_params(self, params, items, label): | 836 def build_list_params(self, params, items, label): |
575 if isinstance(items, str): | 837 if isinstance(items, str): |
576 items = [items] | 838 items = [items] |
577 for i in range(1, len(items) + 1): | 839 for i in range(1, len(items) + 1): |
578 params['%s.%d' % (label, i)] = items[i - 1] | 840 params['%s.%d' % (label, i)] = items[i - 1] |
579 | 841 |
580 # generics | 842 # generics |
581 | 843 |
582 def get_list(self, action, params, markers, path='/', parent=None, verb='GET
'): | 844 def get_list(self, action, params, markers, path='/', |
| 845 parent=None, verb='GET'): |
583 if not parent: | 846 if not parent: |
584 parent = self | 847 parent = self |
585 response = self.make_request(action, params, path, verb) | 848 response = self.make_request(action, params, path, verb) |
586 body = response.read() | 849 body = response.read() |
587 boto.log.debug(body) | 850 boto.log.debug(body) |
588 if not body: | 851 if not body: |
589 boto.log.error('Null body %s' % body) | 852 boto.log.error('Null body %s' % body) |
590 raise self.ResponseError(response.status, response.reason, body) | 853 raise self.ResponseError(response.status, response.reason, body) |
591 elif response.status == 200: | 854 elif response.status == 200: |
592 rs = ResultSet(markers) | 855 rs = ResultSet(markers) |
593 h = handler.XmlHandler(rs, parent) | 856 h = boto.handler.XmlHandler(rs, parent) |
594 xml.sax.parseString(body, h) | 857 xml.sax.parseString(body, h) |
595 return rs | 858 return rs |
596 else: | 859 else: |
597 boto.log.error('%s %s' % (response.status, response.reason)) | 860 boto.log.error('%s %s' % (response.status, response.reason)) |
598 boto.log.error('%s' % body) | 861 boto.log.error('%s' % body) |
599 raise self.ResponseError(response.status, response.reason, body) | 862 raise self.ResponseError(response.status, response.reason, body) |
600 | 863 |
601 def get_object(self, action, params, cls, path='/', parent=None, verb='GET')
: | 864 def get_object(self, action, params, cls, path='/', |
| 865 parent=None, verb='GET'): |
602 if not parent: | 866 if not parent: |
603 parent = self | 867 parent = self |
604 response = self.make_request(action, params, path, verb) | 868 response = self.make_request(action, params, path, verb) |
605 body = response.read() | 869 body = response.read() |
606 boto.log.debug(body) | 870 boto.log.debug(body) |
607 if not body: | 871 if not body: |
608 boto.log.error('Null body %s' % body) | 872 boto.log.error('Null body %s' % body) |
609 raise self.ResponseError(response.status, response.reason, body) | 873 raise self.ResponseError(response.status, response.reason, body) |
610 elif response.status == 200: | 874 elif response.status == 200: |
611 obj = cls(parent) | 875 obj = cls(parent) |
612 h = handler.XmlHandler(obj, parent) | 876 h = boto.handler.XmlHandler(obj, parent) |
613 xml.sax.parseString(body, h) | 877 xml.sax.parseString(body, h) |
614 return obj | 878 return obj |
615 else: | 879 else: |
616 boto.log.error('%s %s' % (response.status, response.reason)) | 880 boto.log.error('%s %s' % (response.status, response.reason)) |
617 boto.log.error('%s' % body) | 881 boto.log.error('%s' % body) |
618 raise self.ResponseError(response.status, response.reason, body) | 882 raise self.ResponseError(response.status, response.reason, body) |
619 | 883 |
620 def get_status(self, action, params, path='/', parent=None, verb='GET'): | 884 def get_status(self, action, params, path='/', parent=None, verb='GET'): |
621 if not parent: | 885 if not parent: |
622 parent = self | 886 parent = self |
623 response = self.make_request(action, params, path, verb) | 887 response = self.make_request(action, params, path, verb) |
624 body = response.read() | 888 body = response.read() |
625 boto.log.debug(body) | 889 boto.log.debug(body) |
626 if not body: | 890 if not body: |
627 boto.log.error('Null body %s' % body) | 891 boto.log.error('Null body %s' % body) |
628 raise self.ResponseError(response.status, response.reason, body) | 892 raise self.ResponseError(response.status, response.reason, body) |
629 elif response.status == 200: | 893 elif response.status == 200: |
630 rs = ResultSet() | 894 rs = ResultSet() |
631 h = handler.XmlHandler(rs, parent) | 895 h = boto.handler.XmlHandler(rs, parent) |
632 xml.sax.parseString(body, h) | 896 xml.sax.parseString(body, h) |
633 return rs.status | 897 return rs.status |
634 else: | 898 else: |
635 boto.log.error('%s %s' % (response.status, response.reason)) | 899 boto.log.error('%s %s' % (response.status, response.reason)) |
636 boto.log.error('%s' % body) | 900 boto.log.error('%s' % body) |
637 raise self.ResponseError(response.status, response.reason, body) | 901 raise self.ResponseError(response.status, response.reason, body) |
OLD | NEW |