Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2016 The LUCI Authors. All rights reserved. | 2 # Copyright 2016 The LUCI Authors. All rights reserved. |
| 3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
| 4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
| 5 | 5 |
| 6 """A low-level blob storage/retrieval interface to the Isolate server""" | 6 """A low-level blob storage/retrieval interface to the Isolate server""" |
| 7 | 7 |
| 8 import base64 | 8 import base64 |
| 9 import binascii | |
| 10 import collections | 9 import collections |
| 11 import logging | 10 import logging |
| 12 import os | 11 import os |
| 13 import re | 12 import re |
| 14 import sys | 13 import sys |
| 15 import threading | 14 import threading |
| 16 import time | 15 import time |
| 17 import types | 16 import types |
| 18 import uuid | 17 import uuid |
| 19 | 18 |
| (...skipping 12 matching lines...) Expand all Loading... | |
| 32 from google.auth.transport import requests as google_auth_transport_requests | 31 from google.auth.transport import requests as google_auth_transport_requests |
| 33 from proto import bytestream_pb2 | 32 from proto import bytestream_pb2 |
| 34 except ImportError as err: | 33 except ImportError as err: |
| 35 grpc = None | 34 grpc = None |
| 36 bytestream_pb2 = None | 35 bytestream_pb2 = None |
| 37 | 36 |
| 38 # If gRPC is installed, at least give a warning if certifi is not. This is not | 37 # If gRPC is installed, at least give a warning if certifi is not. This is not |
| 39 # actually used anywhere in this module, but if certifi is missing, | 38 # actually used anywhere in this module, but if certifi is missing, |
| 40 # google.auth.transport will fail with | 39 # google.auth.transport will fail with |
| 41 # https://stackoverflow.com/questions/24973326 | 40 # https://stackoverflow.com/questions/24973326 |
| 41 certifi = None | |
| 42 if grpc is not None: | 42 if grpc is not None: |
| 43 try: | 43 try: |
| 44 import certifi | 44 import certifi |
| 45 except ImportError as err: | 45 except ImportError as err: |
|
smut
2017/07/13 00:15:36
Unused variable "err".
Vadim Sh.
2017/07/13 00:18:03
Done.
| |
| 46 logging.warning('could not import certifi; gRPC HTTPS connections may fail') | 46 pass # could not import certifi; gRPC HTTPS connections may fail |
| 47 | 47 |
| 48 # Chunk size to use when reading from network stream. | 48 # Chunk size to use when reading from network stream. |
| 49 NET_IO_FILE_CHUNK = 16 * 1024 | 49 NET_IO_FILE_CHUNK = 16 * 1024 |
| 50 | 50 |
| 51 | 51 |
| 52 # Read timeout in seconds for downloads from isolate storage. If there's no | 52 # Read timeout in seconds for downloads from isolate storage. If there's no |
| 53 # response from the server within this timeout whole download will be aborted. | 53 # response from the server within this timeout whole download will be aborted. |
| 54 DOWNLOAD_READ_TIMEOUT = 60 | 54 DOWNLOAD_READ_TIMEOUT = 60 |
| 55 | 55 |
| 56 | 56 |
| (...skipping 466 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 523 class IsolateServerGrpc(StorageApi): | 523 class IsolateServerGrpc(StorageApi): |
| 524 """StorageApi implementation that downloads and uploads to a gRPC service. | 524 """StorageApi implementation that downloads and uploads to a gRPC service. |
| 525 | 525 |
| 526 Limitations: only works for the default-gzip namespace, and with zero offsets | 526 Limitations: only works for the default-gzip namespace, and with zero offsets |
| 527 while fetching. | 527 while fetching. |
| 528 """ | 528 """ |
| 529 | 529 |
| 530 def __init__(self, server, namespace, proxy): | 530 def __init__(self, server, namespace, proxy): |
| 531 super(IsolateServerGrpc, self).__init__() | 531 super(IsolateServerGrpc, self).__init__() |
| 532 logging.info('Using gRPC for Isolate') | 532 logging.info('Using gRPC for Isolate') |
| 533 if not certifi: | |
| 534 logging.warning( | |
| 535 'Could not import certifi; gRPC HTTPS connections may fail') | |
| 533 self._server = server | 536 self._server = server |
| 534 self._lock = threading.Lock() | 537 self._lock = threading.Lock() |
| 535 self._memory_use = 0 | 538 self._memory_use = 0 |
| 536 self._num_pushes = 0 | 539 self._num_pushes = 0 |
| 537 self._already_exists = 0 | 540 self._already_exists = 0 |
| 538 | 541 |
| 539 # Proxies only support the default-gzip namespace for now. | 542 # Proxies only support the default-gzip namespace for now. |
| 540 # TODO(aludwin): support other namespaces if necessary | 543 # TODO(aludwin): support other namespaces if necessary |
| 541 assert namespace == 'default-gzip' | 544 assert namespace == 'default-gzip' |
| 542 self._namespace = namespace | 545 self._namespace = namespace |
| (...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 716 namespace: isolate namespace to operate in, also defines hashing and | 719 namespace: isolate namespace to operate in, also defines hashing and |
| 717 compression scheme used, i.e. namespace names that end with '-gzip' | 720 compression scheme used, i.e. namespace names that end with '-gzip' |
| 718 store compressed data. | 721 store compressed data. |
| 719 | 722 |
| 720 Returns: | 723 Returns: |
| 721 Instance of StorageApi subclass. | 724 Instance of StorageApi subclass. |
| 722 """ | 725 """ |
| 723 if _grpc_proxy is not None: | 726 if _grpc_proxy is not None: |
| 724 return IsolateServerGrpc(url, namespace, _grpc_proxy) | 727 return IsolateServerGrpc(url, namespace, _grpc_proxy) |
| 725 return IsolateServer(url, namespace) | 728 return IsolateServer(url, namespace) |
| OLD | NEW |