| OLD | NEW |
| 1 # Copyright 2017 The Chromium Authors. All rights reserved. | 1 # Copyright 2017 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Helper functions to upload data to Google Storage. | 5 """Helper functions to upload data to Google Storage. |
| 6 | 6 |
| 7 Text data should be streamed to logdog using |logdog_helper| module. | 7 Text data should be streamed to logdog using |logdog_helper| module. |
| 8 Due to logdog not having image or HTML viewer, those instead should be uploaded | 8 Due to logdog not having image or HTML viewer, those instead should be uploaded |
| 9 to Google Storage directly using this module. | 9 to Google Storage directly using this module. |
| 10 """ | 10 """ |
| 11 | 11 |
| 12 import hashlib | |
| 13 import logging | 12 import logging |
| 14 import os | 13 import os |
| 15 import sys | 14 import sys |
| 16 import time | 15 import time |
| 17 | 16 |
| 18 from pylib.constants import host_paths | 17 from pylib.constants import host_paths |
| 19 from pylib.utils import decorators | 18 from pylib.utils import decorators |
| 20 | 19 |
| 21 if host_paths.DEVIL_PATH not in sys.path: | 20 if host_paths.DEVIL_PATH not in sys.path: |
| 22 sys.path.append(host_paths.DEVIL_PATH) | 21 sys.path.append(host_paths.DEVIL_PATH) |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 54 cmd = [_GSUTIL_PATH, '-q'] | 53 cmd = [_GSUTIL_PATH, '-q'] |
| 55 if content_type: | 54 if content_type: |
| 56 cmd.extend(['-h', 'Content-Type:%s' % content_type]) | 55 cmd.extend(['-h', 'Content-Type:%s' % content_type]) |
| 57 cmd.extend(['cp', filepath, gs_path]) | 56 cmd.extend(['cp', filepath, gs_path]) |
| 58 | 57 |
| 59 cmd_helper.RunCmd(cmd) | 58 cmd_helper.RunCmd(cmd) |
| 60 | 59 |
| 61 return get_url_link(name, bucket, authenticated_link) | 60 return get_url_link(name, bucket, authenticated_link) |
| 62 | 61 |
| 63 | 62 |
| 64 def upload_content_addressed( | |
| 65 filepath, bucket, content_type=None, authenticated_link=True): | |
| 66 """Uploads data to Google Storage with filename as sha1 hash. | |
| 67 | |
| 68 If file already exists in bucket with hash name, nothing is uploaded. | |
| 69 """ | |
| 70 sha1 = hashlib.sha1() | |
| 71 with open(filepath, 'rb') as f: | |
| 72 sha1.update(f.read()) | |
| 73 sha1_hash = sha1.hexdigest() | |
| 74 if not exists(sha1_hash, bucket): | |
| 75 upload(sha1_hash, filepath, bucket, content_type, authenticated_link) | |
| 76 return get_url_link(sha1_hash, bucket, authenticated_link) | |
| 77 | |
| 78 | |
| 79 @decorators.NoRaiseException(default_return_value=False) | 63 @decorators.NoRaiseException(default_return_value=False) |
| 80 def exists(name, bucket): | 64 def exists(name, bucket): |
| 81 bucket = _format_bucket_name(bucket) | 65 bucket = _format_bucket_name(bucket) |
| 82 gs_path = 'gs://%s/%s' % (bucket, name) | 66 gs_path = 'gs://%s/%s' % (bucket, name) |
| 83 | 67 |
| 84 cmd = [_GSUTIL_PATH, '-q', 'stat', gs_path] | 68 cmd = [_GSUTIL_PATH, '-q', 'stat', gs_path] |
| 85 return_code = cmd_helper.RunCmd(cmd) | 69 return_code = cmd_helper.RunCmd(cmd) |
| 86 if return_code == 0: | 70 if return_code == 0: |
| 87 return True | 71 return True |
| 88 else: | 72 else: |
| 89 return False | 73 return False |
| 90 | 74 |
| 91 | 75 |
| 76 # TODO(jbudorick): Delete this function. Only one user of it. |
| 92 def unique_name(basename, suffix='', timestamp=True, device=None): | 77 def unique_name(basename, suffix='', timestamp=True, device=None): |
| 93 """Helper function for creating a unique name for a file to store in GS. | 78 """Helper function for creating a unique name for a file to store in GS. |
| 94 | 79 |
| 95 Args: | 80 Args: |
| 96 basename: Base of the unique filename. | 81 basename: Base of the unique filename. |
| 97 suffix: Suffix of filename. | 82 suffix: Suffix of filename. |
| 98 timestamp: Whether or not to add a timestamp to name. | 83 timestamp: Whether or not to add a timestamp to name. |
| 99 device: Device to add device serial of to name. | 84 device: Device to add device serial of to name. |
| 100 """ | 85 """ |
| 101 return '%s%s%s%s' % ( | 86 return '%s%s%s%s' % ( |
| (...skipping 21 matching lines...) Expand all Loading... |
| 123 url_template = _AUTHENTICATED_URL if authenticated_link else _PUBLIC_URL | 108 url_template = _AUTHENTICATED_URL if authenticated_link else _PUBLIC_URL |
| 124 return os.path.join(url_template % bucket, name) | 109 return os.path.join(url_template % bucket, name) |
| 125 | 110 |
| 126 | 111 |
| 127 def _format_bucket_name(bucket): | 112 def _format_bucket_name(bucket): |
| 128 if bucket.startswith('gs://'): | 113 if bucket.startswith('gs://'): |
| 129 bucket = bucket[len('gs://'):] | 114 bucket = bucket[len('gs://'):] |
| 130 if bucket.endswith('/'): | 115 if bucket.endswith('/'): |
| 131 bucket = bucket[:-1] | 116 bucket = bucket[:-1] |
| 132 return bucket | 117 return bucket |
| OLD | NEW |