| OLD | NEW |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import re | 5 import re |
| 6 | 6 |
| 7 from recipe_engine import recipe_api | 7 from recipe_engine import recipe_api |
| 8 | 8 |
| 9 class GSUtilApi(recipe_api.RecipeApi): | 9 class GSUtilApi(recipe_api.RecipeApi): |
| 10 def __call__(self, cmd, name=None, use_retry_wrapper=True, version='3.25', | 10 def __call__(self, cmd, name=None, use_retry_wrapper=True, version='3.25', |
| (...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 81 args = args or [] | 81 args = args or [] |
| 82 url = self._normalize_url(url) | 82 url = self._normalize_url(url) |
| 83 cmd = ['cp'] + args + [url, dest] | 83 cmd = ['cp'] + args + [url, dest] |
| 84 name = kwargs.pop('name', 'download') | 84 name = kwargs.pop('name', 'download') |
| 85 self(cmd, name, **kwargs) | 85 self(cmd, name, **kwargs) |
| 86 | 86 |
| 87 def cat(self, url, args=None, **kwargs): | 87 def cat(self, url, args=None, **kwargs): |
| 88 args = args or [] | 88 args = args or [] |
| 89 url = self._normalize_url(url) | 89 url = self._normalize_url(url) |
| 90 cmd = ['cat'] + args + [url] | 90 cmd = ['cat'] + args + [url] |
| 91 name = kwargs.pop('name', 'download') | 91 name = kwargs.pop('name', 'cat') |
| 92 return self(cmd, name, **kwargs) | 92 return self(cmd, name, **kwargs) |
| 93 | 93 |
| 94 def copy(self, source_bucket, source, dest_bucket, dest, args=None, | 94 def copy(self, source_bucket, source, dest_bucket, dest, args=None, |
| 95 link_name='gsutil.copy', metadata=None, unauthenticated_url=False, | 95 link_name='gsutil.copy', metadata=None, unauthenticated_url=False, |
| 96 **kwargs): | 96 **kwargs): |
| 97 args = args or [] | 97 args = args or [] |
| 98 args += self._generate_metadata_args(metadata) | 98 args += self._generate_metadata_args(metadata) |
| 99 full_source = 'gs://%s/%s' % (source_bucket, source) | 99 full_source = 'gs://%s/%s' % (source_bucket, source) |
| 100 full_dest = 'gs://%s/%s' % (dest_bucket, dest) | 100 full_dest = 'gs://%s/%s' % (dest_bucket, dest) |
| 101 cmd = ['cp'] + args + [full_source, full_dest] | 101 cmd = ['cp'] + args + [full_source, full_dest] |
| 102 name = kwargs.pop('name', 'copy') | 102 name = kwargs.pop('name', 'copy') |
| 103 | 103 |
| 104 result = self(cmd, name, **kwargs) | 104 result = self(cmd, name, **kwargs) |
| 105 | 105 |
| 106 if link_name: | 106 if link_name: |
| 107 result.presentation.links[link_name] = self._http_url( | 107 result.presentation.links[link_name] = self._http_url( |
| 108 dest_bucket, dest, unauthenticated_url=unauthenticated_url) | 108 dest_bucket, dest, unauthenticated_url=unauthenticated_url) |
| 109 | 109 |
| 110 def list(self, url, args=None, **kwargs): |
| 111 args = args or [] |
| 112 url = self._normalize_url(url) |
| 113 cmd = ['ls'] + args + [url] |
| 114 name = kwargs.pop('name', 'list') |
| 115 return self(cmd, name, **kwargs) |
| 116 |
| 110 def signurl(self, private_key_file, bucket, dest, args=None, | 117 def signurl(self, private_key_file, bucket, dest, args=None, |
| 111 **kwargs): | 118 **kwargs): |
| 112 args = args or [] | 119 args = args or [] |
| 113 full_source = 'gs://%s/%s' % (bucket, dest) | 120 full_source = 'gs://%s/%s' % (bucket, dest) |
| 114 cmd = ['signurl'] + args + [private_key_file, full_source] | 121 cmd = ['signurl'] + args + [private_key_file, full_source] |
| 115 name = kwargs.pop('name', 'signurl') | 122 name = kwargs.pop('name', 'signurl') |
| 116 return self(cmd, name, **kwargs) | 123 return self(cmd, name, **kwargs) |
| 117 | 124 |
| 118 def remove_url(self, url, args=None, **kwargs): | 125 def remove_url(self, url, args=None, **kwargs): |
| 119 args = args or [] | 126 args = args or [] |
| (...skipping 21 matching lines...) Expand all Loading... |
| 141 args = ['--poll', | 148 args = ['--poll', |
| 142 '--url', url, | 149 '--url', url, |
| 143 '--dst', destination, | 150 '--dst', destination, |
| 144 '--poll-interval', str(poll_interval), | 151 '--poll-interval', str(poll_interval), |
| 145 '--timeout', str(timeout)] | 152 '--timeout', str(timeout)] |
| 146 return self.m.python(name, | 153 return self.m.python(name, |
| 147 gsutil_download_path, | 154 gsutil_download_path, |
| 148 args, | 155 args, |
| 149 cwd=self.m.path['slave_build']) | 156 cwd=self.m.path['slave_build']) |
| 150 | 157 |
| 158 def download_latest_file(self, base_url, partial_name, destination, |
| 159 name='Download latest file from GS'): |
| 160 """Get the latest archived object with the given base url and partial name. |
| 161 |
| 162 Args: |
| 163 base_url: Base Google Storage archive URL (gs://...) containing the build. |
| 164 partial_name: Partial name of the archive file to download. |
| 165 destination: Destination file/directory where the file will be downloaded. |
| 166 name: The name of the step. |
| 167 """ |
| 168 gsutil_download_path = self.m.path['build'].join( |
| 169 'scripts', 'slave', 'gsutil_download.py') |
| 170 args = ['--url', base_url, |
| 171 '--dst', destination, |
| 172 '--partial-name', partial_name] |
| 173 return self.m.python(name, |
| 174 gsutil_download_path, |
| 175 args, |
| 176 cwd=self.m.path['slave_build']) |
| 177 |
| 151 def _generate_metadata_args(self, metadata): | 178 def _generate_metadata_args(self, metadata): |
| 152 result = [] | 179 result = [] |
| 153 if metadata: | 180 if metadata: |
| 154 for k, v in sorted(metadata.iteritems(), key=lambda (k, _): k): | 181 for k, v in sorted(metadata.iteritems(), key=lambda (k, _): k): |
| 155 field = self._get_metadata_field(k) | 182 field = self._get_metadata_field(k) |
| 156 param = (field) if v is None else ('%s:%s' % (field, v)) | 183 param = (field) if v is None else ('%s:%s' % (field, v)) |
| 157 result += ['-h', param] | 184 result += ['-h', param] |
| 158 return result | 185 return result |
| 159 | 186 |
| 160 def _normalize_url(self, url): | 187 def _normalize_url(self, url): |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 196 'Content-Language', | 223 'Content-Language', |
| 197 'Content-MD5', | 224 'Content-MD5', |
| 198 'Content-Type', | 225 'Content-Type', |
| 199 ): | 226 ): |
| 200 return name | 227 return name |
| 201 | 228 |
| 202 # Add provider prefix | 229 # Add provider prefix |
| 203 if not provider_prefix: | 230 if not provider_prefix: |
| 204 provider_prefix = 'x-goog-meta' | 231 provider_prefix = 'x-goog-meta' |
| 205 return '%s-%s' % (provider_prefix, name) | 232 return '%s-%s' % (provider_prefix, name) |
| OLD | NEW |