OLD | NEW |
1 # Copyright 2017 The Chromium Authors. All rights reserved. | 1 # Copyright 2017 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Helper functions to upload data to Google Storage. | 5 """Helper functions to upload data to Google Storage. |
6 | 6 |
7 Text data should be streamed to logdog using |logdog_helper| module. | 7 Text data should be streamed to logdog using |logdog_helper| module. |
8 Due to logdog not having image or HTML viewer, those instead should be uploaded | 8 Due to logdog not having image or HTML viewer, those instead should be uploaded |
9 to Google Storage directly using this module. | 9 to Google Storage directly using this module. |
10 """ | 10 """ |
11 | 11 |
12 import hashlib | |
13 import logging | 12 import logging |
14 import os | 13 import os |
15 import sys | 14 import sys |
16 import time | 15 import time |
17 | 16 |
18 from pylib.constants import host_paths | 17 from pylib.constants import host_paths |
19 from pylib.utils import decorators | 18 from pylib.utils import decorators |
20 | 19 |
21 if host_paths.DEVIL_PATH not in sys.path: | 20 if host_paths.DEVIL_PATH not in sys.path: |
22 sys.path.append(host_paths.DEVIL_PATH) | 21 sys.path.append(host_paths.DEVIL_PATH) |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
54 cmd = [_GSUTIL_PATH, '-q'] | 53 cmd = [_GSUTIL_PATH, '-q'] |
55 if content_type: | 54 if content_type: |
56 cmd.extend(['-h', 'Content-Type:%s' % content_type]) | 55 cmd.extend(['-h', 'Content-Type:%s' % content_type]) |
57 cmd.extend(['cp', filepath, gs_path]) | 56 cmd.extend(['cp', filepath, gs_path]) |
58 | 57 |
59 cmd_helper.RunCmd(cmd) | 58 cmd_helper.RunCmd(cmd) |
60 | 59 |
61 return get_url_link(name, bucket, authenticated_link) | 60 return get_url_link(name, bucket, authenticated_link) |
62 | 61 |
63 | 62 |
64 def upload_content_addressed( | |
65 filepath, bucket, content_type=None, authenticated_link=True): | |
66 """Uploads data to Google Storage with filename as sha1 hash. | |
67 | |
68 If file already exists in bucket with hash name, nothing is uploaded. | |
69 """ | |
70 sha1 = hashlib.sha1() | |
71 with open(filepath, 'rb') as f: | |
72 sha1.update(f.read()) | |
73 sha1_hash = sha1.hexdigest() | |
74 if not exists(sha1_hash, bucket): | |
75 upload(sha1_hash, filepath, bucket, content_type, authenticated_link) | |
76 return get_url_link(sha1_hash, bucket, authenticated_link) | |
77 | |
78 | |
79 @decorators.NoRaiseException(default_return_value=False) | 63 @decorators.NoRaiseException(default_return_value=False) |
80 def exists(name, bucket): | 64 def exists(name, bucket): |
81 bucket = _format_bucket_name(bucket) | 65 bucket = _format_bucket_name(bucket) |
82 gs_path = 'gs://%s/%s' % (bucket, name) | 66 gs_path = 'gs://%s/%s' % (bucket, name) |
83 | 67 |
84 cmd = [_GSUTIL_PATH, '-q', 'stat', gs_path] | 68 cmd = [_GSUTIL_PATH, '-q', 'stat', gs_path] |
85 return_code = cmd_helper.RunCmd(cmd) | 69 return_code = cmd_helper.RunCmd(cmd) |
86 if return_code == 0: | 70 if return_code == 0: |
87 return True | 71 return True |
88 else: | 72 else: |
89 return False | 73 return False |
90 | 74 |
91 | 75 |
| 76 # TODO(jbudorick): Delete this function. Only one user of it. |
92 def unique_name(basename, suffix='', timestamp=True, device=None): | 77 def unique_name(basename, suffix='', timestamp=True, device=None): |
93 """Helper function for creating a unique name for a file to store in GS. | 78 """Helper function for creating a unique name for a file to store in GS. |
94 | 79 |
95 Args: | 80 Args: |
96 basename: Base of the unique filename. | 81 basename: Base of the unique filename. |
97 suffix: Suffix of filename. | 82 suffix: Suffix of filename. |
98 timestamp: Whether or not to add a timestamp to name. | 83 timestamp: Whether or not to add a timestamp to name. |
99 device: Device to add device serial of to name. | 84 device: Device to add device serial of to name. |
100 """ | 85 """ |
101 return '%s%s%s%s' % ( | 86 return '%s%s%s%s' % ( |
(...skipping 21 matching lines...) Expand all Loading... |
123 url_template = _AUTHENTICATED_URL if authenticated_link else _PUBLIC_URL | 108 url_template = _AUTHENTICATED_URL if authenticated_link else _PUBLIC_URL |
124 return os.path.join(url_template % bucket, name) | 109 return os.path.join(url_template % bucket, name) |
125 | 110 |
126 | 111 |
127 def _format_bucket_name(bucket): | 112 def _format_bucket_name(bucket): |
128 if bucket.startswith('gs://'): | 113 if bucket.startswith('gs://'): |
129 bucket = bucket[len('gs://'):] | 114 bucket = bucket[len('gs://'):] |
130 if bucket.endswith('/'): | 115 if bucket.endswith('/'): |
131 bucket = bucket[:-1] | 116 bucket = bucket[:-1] |
132 return bucket | 117 return bucket |
OLD | NEW |