OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2016 The LUCI Authors. All rights reserved. | 2 # Copyright 2016 The LUCI Authors. All rights reserved. |
3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
5 | 5 |
6 """A low-level blob storage/retrieval interface to the Isolate server""" | 6 """A low-level blob storage/retrieval interface to the Isolate server""" |
7 | 7 |
8 import base64 | 8 import base64 |
9 import binascii | |
10 import collections | 9 import collections |
11 import logging | 10 import logging |
12 import os | 11 import os |
13 import re | 12 import re |
14 import sys | 13 import sys |
15 import threading | 14 import threading |
16 import time | 15 import time |
17 import types | 16 import types |
18 import uuid | 17 import uuid |
19 | 18 |
(...skipping 12 matching lines...) Expand all Loading... | |
32 from google.auth.transport import requests as google_auth_transport_requests | 31 from google.auth.transport import requests as google_auth_transport_requests |
33 from proto import bytestream_pb2 | 32 from proto import bytestream_pb2 |
34 except ImportError as err: | 33 except ImportError as err: |
35 grpc = None | 34 grpc = None |
36 bytestream_pb2 = None | 35 bytestream_pb2 = None |
37 | 36 |
38 # If gRPC is installed, at least give a warning if certifi is not. This is not | 37 # If gRPC is installed, at least give a warning if certifi is not. This is not |
39 # actually used anywhere in this module, but if certifi is missing, | 38 # actually used anywhere in this module, but if certifi is missing, |
40 # google.auth.transport will fail with | 39 # google.auth.transport will fail with |
41 # https://stackoverflow.com/questions/24973326 | 40 # https://stackoverflow.com/questions/24973326 |
41 certifi = None | |
42 if grpc is not None: | 42 if grpc is not None: |
43 try: | 43 try: |
44 import certifi | 44 import certifi |
45 except ImportError as err: | 45 except ImportError as err: |
smut
2017/07/13 00:15:36
Unused variable "err".
Vadim Sh.
2017/07/13 00:18:03
Done.
| |
46 logging.warning('could not import certifi; gRPC HTTPS connections may fail') | 46 pass # could not import certifi; gRPC HTTPS connections may fail |
47 | 47 |
48 # Chunk size to use when reading from network stream. | 48 # Chunk size to use when reading from network stream. |
49 NET_IO_FILE_CHUNK = 16 * 1024 | 49 NET_IO_FILE_CHUNK = 16 * 1024 |
50 | 50 |
51 | 51 |
52 # Read timeout in seconds for downloads from isolate storage. If there's no | 52 # Read timeout in seconds for downloads from isolate storage. If there's no |
53 # response from the server within this timeout whole download will be aborted. | 53 # response from the server within this timeout whole download will be aborted. |
54 DOWNLOAD_READ_TIMEOUT = 60 | 54 DOWNLOAD_READ_TIMEOUT = 60 |
55 | 55 |
56 | 56 |
(...skipping 466 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
523 class IsolateServerGrpc(StorageApi): | 523 class IsolateServerGrpc(StorageApi): |
524 """StorageApi implementation that downloads and uploads to a gRPC service. | 524 """StorageApi implementation that downloads and uploads to a gRPC service. |
525 | 525 |
526 Limitations: only works for the default-gzip namespace, and with zero offsets | 526 Limitations: only works for the default-gzip namespace, and with zero offsets |
527 while fetching. | 527 while fetching. |
528 """ | 528 """ |
529 | 529 |
530 def __init__(self, server, namespace, proxy): | 530 def __init__(self, server, namespace, proxy): |
531 super(IsolateServerGrpc, self).__init__() | 531 super(IsolateServerGrpc, self).__init__() |
532 logging.info('Using gRPC for Isolate') | 532 logging.info('Using gRPC for Isolate') |
533 if not certifi: | |
534 logging.warning( | |
535 'Could not import certifi; gRPC HTTPS connections may fail') | |
533 self._server = server | 536 self._server = server |
534 self._lock = threading.Lock() | 537 self._lock = threading.Lock() |
535 self._memory_use = 0 | 538 self._memory_use = 0 |
536 self._num_pushes = 0 | 539 self._num_pushes = 0 |
537 self._already_exists = 0 | 540 self._already_exists = 0 |
538 | 541 |
539 # Proxies only support the default-gzip namespace for now. | 542 # Proxies only support the default-gzip namespace for now. |
540 # TODO(aludwin): support other namespaces if necessary | 543 # TODO(aludwin): support other namespaces if necessary |
541 assert namespace == 'default-gzip' | 544 assert namespace == 'default-gzip' |
542 self._namespace = namespace | 545 self._namespace = namespace |
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
716 namespace: isolate namespace to operate in, also defines hashing and | 719 namespace: isolate namespace to operate in, also defines hashing and |
717 compression scheme used, i.e. namespace names that end with '-gzip' | 720 compression scheme used, i.e. namespace names that end with '-gzip' |
718 store compressed data. | 721 store compressed data. |
719 | 722 |
720 Returns: | 723 Returns: |
721 Instance of StorageApi subclass. | 724 Instance of StorageApi subclass. |
722 """ | 725 """ |
723 if _grpc_proxy is not None: | 726 if _grpc_proxy is not None: |
724 return IsolateServerGrpc(url, namespace, _grpc_proxy) | 727 return IsolateServerGrpc(url, namespace, _grpc_proxy) |
725 return IsolateServer(url, namespace) | 728 return IsolateServer(url, namespace) |
OLD | NEW |