| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2016 The LUCI Authors. All rights reserved. | 2 # Copyright 2016 The LUCI Authors. All rights reserved. |
| 3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
| 4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
| 5 | 5 |
| 6 """A low-level blob storage/retrieval interface to the Isolate server""" | 6 """A low-level blob storage/retrieval interface to the Isolate server""" |
| 7 | 7 |
| 8 import base64 | 8 import base64 |
| 9 import binascii | |
| 10 import collections | 9 import collections |
| 11 import logging | 10 import logging |
| 12 import os | 11 import os |
| 13 import re | 12 import re |
| 14 import sys | 13 import sys |
| 15 import threading | 14 import threading |
| 16 import time | 15 import time |
| 17 import types | 16 import types |
| 18 import uuid | 17 import uuid |
| 19 | 18 |
| (...skipping 12 matching lines...) Expand all Loading... |
| 32 from google.auth.transport import requests as google_auth_transport_requests | 31 from google.auth.transport import requests as google_auth_transport_requests |
| 33 from proto import bytestream_pb2 | 32 from proto import bytestream_pb2 |
| 34 except ImportError as err: | 33 except ImportError as err: |
| 35 grpc = None | 34 grpc = None |
| 36 bytestream_pb2 = None | 35 bytestream_pb2 = None |
| 37 | 36 |
| 38 # If gRPC is installed, at least give a warning if certifi is not. This is not | 37 # If gRPC is installed, at least give a warning if certifi is not. This is not |
| 39 # actually used anywhere in this module, but if certifi is missing, | 38 # actually used anywhere in this module, but if certifi is missing, |
| 40 # google.auth.transport will fail with | 39 # google.auth.transport will fail with |
| 41 # https://stackoverflow.com/questions/24973326 | 40 # https://stackoverflow.com/questions/24973326 |
| 41 certifi = None |
| 42 if grpc is not None: | 42 if grpc is not None: |
| 43 try: | 43 try: |
| 44 import certifi | 44 import certifi |
| 45 except ImportError as err: | 45 except ImportError: |
| 46 logging.warning('could not import certifi; gRPC HTTPS connections may fail') | 46 # Could not import certifi; gRPC HTTPS connections may fail. This will be |
| 47 # logged in IsolateServerGrpc.__init__, since the logger is not configured |
| 48 # during the import time. |
| 49 pass |
| 47 | 50 |
| 48 # Chunk size to use when reading from network stream. | 51 # Chunk size to use when reading from network stream. |
| 49 NET_IO_FILE_CHUNK = 16 * 1024 | 52 NET_IO_FILE_CHUNK = 16 * 1024 |
| 50 | 53 |
| 51 | 54 |
| 52 # Read timeout in seconds for downloads from isolate storage. If there's no | 55 # Read timeout in seconds for downloads from isolate storage. If there's no |
| 53 # response from the server within this timeout whole download will be aborted. | 56 # response from the server within this timeout whole download will be aborted. |
| 54 DOWNLOAD_READ_TIMEOUT = 60 | 57 DOWNLOAD_READ_TIMEOUT = 60 |
| 55 | 58 |
| 56 | 59 |
| (...skipping 466 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 523 class IsolateServerGrpc(StorageApi): | 526 class IsolateServerGrpc(StorageApi): |
| 524 """StorageApi implementation that downloads and uploads to a gRPC service. | 527 """StorageApi implementation that downloads and uploads to a gRPC service. |
| 525 | 528 |
| 526 Limitations: only works for the default-gzip namespace, and with zero offsets | 529 Limitations: only works for the default-gzip namespace, and with zero offsets |
| 527 while fetching. | 530 while fetching. |
| 528 """ | 531 """ |
| 529 | 532 |
| 530 def __init__(self, server, namespace, proxy): | 533 def __init__(self, server, namespace, proxy): |
| 531 super(IsolateServerGrpc, self).__init__() | 534 super(IsolateServerGrpc, self).__init__() |
| 532 logging.info('Using gRPC for Isolate') | 535 logging.info('Using gRPC for Isolate') |
| 536 if not certifi: |
| 537 logging.warning( |
| 538 'Could not import certifi; gRPC HTTPS connections may fail') |
| 533 self._server = server | 539 self._server = server |
| 534 self._lock = threading.Lock() | 540 self._lock = threading.Lock() |
| 535 self._memory_use = 0 | 541 self._memory_use = 0 |
| 536 self._num_pushes = 0 | 542 self._num_pushes = 0 |
| 537 self._already_exists = 0 | 543 self._already_exists = 0 |
| 538 | 544 |
| 539 # Proxies only support the default-gzip namespace for now. | 545 # Proxies only support the default-gzip namespace for now. |
| 540 # TODO(aludwin): support other namespaces if necessary | 546 # TODO(aludwin): support other namespaces if necessary |
| 541 assert namespace == 'default-gzip' | 547 assert namespace == 'default-gzip' |
| 542 self._namespace = namespace | 548 self._namespace = namespace |
| (...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 716 namespace: isolate namespace to operate in, also defines hashing and | 722 namespace: isolate namespace to operate in, also defines hashing and |
| 717 compression scheme used, i.e. namespace names that end with '-gzip' | 723 compression scheme used, i.e. namespace names that end with '-gzip' |
| 718 store compressed data. | 724 store compressed data. |
| 719 | 725 |
| 720 Returns: | 726 Returns: |
| 721 Instance of StorageApi subclass. | 727 Instance of StorageApi subclass. |
| 722 """ | 728 """ |
| 723 if _grpc_proxy is not None: | 729 if _grpc_proxy is not None: |
| 724 return IsolateServerGrpc(url, namespace, _grpc_proxy) | 730 return IsolateServerGrpc(url, namespace, _grpc_proxy) |
| 725 return IsolateServer(url, namespace) | 731 return IsolateServer(url, namespace) |
| OLD | NEW |