OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2016 The LUCI Authors. All rights reserved. | 2 # Copyright 2016 The LUCI Authors. All rights reserved. |
3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
5 | 5 |
6 """A low-level blob storage/retrieval interface to the Isolate server""" | 6 """A low-level blob storage/retrieval interface to the Isolate server""" |
7 | 7 |
8 import base64 | 8 import base64 |
9 import binascii | |
10 import collections | 9 import collections |
11 import logging | 10 import logging |
12 import os | 11 import os |
13 import re | 12 import re |
14 import sys | 13 import sys |
15 import threading | 14 import threading |
16 import time | 15 import time |
17 import types | 16 import types |
18 import uuid | 17 import uuid |
19 | 18 |
(...skipping 12 matching lines...) Expand all Loading... |
32 from google.auth.transport import requests as google_auth_transport_requests | 31 from google.auth.transport import requests as google_auth_transport_requests |
33 from proto import bytestream_pb2 | 32 from proto import bytestream_pb2 |
34 except ImportError as err: | 33 except ImportError as err: |
35 grpc = None | 34 grpc = None |
36 bytestream_pb2 = None | 35 bytestream_pb2 = None |
37 | 36 |
38 # If gRPC is installed, at least give a warning if certifi is not. This is not | 37 # If gRPC is installed, at least give a warning if certifi is not. This is not |
39 # actually used anywhere in this module, but if certifi is missing, | 38 # actually used anywhere in this module, but if certifi is missing, |
40 # google.auth.transport will fail with | 39 # google.auth.transport will fail with |
41 # https://stackoverflow.com/questions/24973326 | 40 # https://stackoverflow.com/questions/24973326 |
| 41 certifi = None |
42 if grpc is not None: | 42 if grpc is not None: |
43 try: | 43 try: |
44 import certifi | 44 import certifi |
45 except ImportError as err: | 45 except ImportError: |
46 logging.warning('could not import certifi; gRPC HTTPS connections may fail') | 46 # Could not import certifi; gRPC HTTPS connections may fail. This will be |
| 47 # logged in IsolateServerGrpc.__init__, since the logger is not configured |
| 48 # during the import time. |
| 49 pass |
47 | 50 |
48 # Chunk size to use when reading from network stream. | 51 # Chunk size to use when reading from network stream. |
49 NET_IO_FILE_CHUNK = 16 * 1024 | 52 NET_IO_FILE_CHUNK = 16 * 1024 |
50 | 53 |
51 | 54 |
52 # Read timeout in seconds for downloads from isolate storage. If there's no | 55 # Read timeout in seconds for downloads from isolate storage. If there's no |
53 # response from the server within this timeout whole download will be aborted. | 56 # response from the server within this timeout whole download will be aborted. |
54 DOWNLOAD_READ_TIMEOUT = 60 | 57 DOWNLOAD_READ_TIMEOUT = 60 |
55 | 58 |
56 | 59 |
(...skipping 466 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
523 class IsolateServerGrpc(StorageApi): | 526 class IsolateServerGrpc(StorageApi): |
524 """StorageApi implementation that downloads and uploads to a gRPC service. | 527 """StorageApi implementation that downloads and uploads to a gRPC service. |
525 | 528 |
526 Limitations: only works for the default-gzip namespace, and with zero offsets | 529 Limitations: only works for the default-gzip namespace, and with zero offsets |
527 while fetching. | 530 while fetching. |
528 """ | 531 """ |
529 | 532 |
530 def __init__(self, server, namespace, proxy): | 533 def __init__(self, server, namespace, proxy): |
531 super(IsolateServerGrpc, self).__init__() | 534 super(IsolateServerGrpc, self).__init__() |
532 logging.info('Using gRPC for Isolate') | 535 logging.info('Using gRPC for Isolate') |
| 536 if not certifi: |
| 537 logging.warning( |
| 538 'Could not import certifi; gRPC HTTPS connections may fail') |
533 self._server = server | 539 self._server = server |
534 self._lock = threading.Lock() | 540 self._lock = threading.Lock() |
535 self._memory_use = 0 | 541 self._memory_use = 0 |
536 self._num_pushes = 0 | 542 self._num_pushes = 0 |
537 self._already_exists = 0 | 543 self._already_exists = 0 |
538 | 544 |
539 # Proxies only support the default-gzip namespace for now. | 545 # Proxies only support the default-gzip namespace for now. |
540 # TODO(aludwin): support other namespaces if necessary | 546 # TODO(aludwin): support other namespaces if necessary |
541 assert namespace == 'default-gzip' | 547 assert namespace == 'default-gzip' |
542 self._namespace = namespace | 548 self._namespace = namespace |
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
716 namespace: isolate namespace to operate in, also defines hashing and | 722 namespace: isolate namespace to operate in, also defines hashing and |
717 compression scheme used, i.e. namespace names that end with '-gzip' | 723 compression scheme used, i.e. namespace names that end with '-gzip' |
718 store compressed data. | 724 store compressed data. |
719 | 725 |
720 Returns: | 726 Returns: |
721 Instance of StorageApi subclass. | 727 Instance of StorageApi subclass. |
722 """ | 728 """ |
723 if _grpc_proxy is not None: | 729 if _grpc_proxy is not None: |
724 return IsolateServerGrpc(url, namespace, _grpc_proxy) | 730 return IsolateServerGrpc(url, namespace, _grpc_proxy) |
725 return IsolateServer(url, namespace) | 731 return IsolateServer(url, namespace) |
OLD | NEW |