| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 import itertools | |
| 6 import re | |
| 7 | |
| 8 from recipe_engine import recipe_api | |
| 9 | |
| 10 class GitApi(recipe_api.RecipeApi): | |
| 11 _GIT_HASH_RE = re.compile('[0-9a-f]{40}', re.IGNORECASE) | |
| 12 | |
| 13 def __call__(self, *args, **kwargs): | |
| 14 """Return a git command step.""" | |
| 15 name = kwargs.pop('name', 'git '+args[0]) | |
| 16 infra_step = kwargs.pop('infra_step', True) | |
| 17 if 'cwd' not in kwargs: | |
| 18 kwargs.setdefault('cwd', self.m.path['checkout']) | |
| 19 git_cmd = ['git'] | |
| 20 if self.m.platform.is_win: | |
| 21 git_cmd = [self.m.path['depot_tools'].join('git.bat')] | |
| 22 options = kwargs.pop('git_config_options', {}) | |
| 23 for k, v in sorted(options.iteritems()): | |
| 24 git_cmd.extend(['-c', '%s=%s' % (k, v)]) | |
| 25 can_fail_build = kwargs.pop('can_fail_build', True) | |
| 26 try: | |
| 27 return self.m.step(name, git_cmd + list(args), infra_step=infra_step, | |
| 28 **kwargs) | |
| 29 except self.m.step.StepFailure as f: | |
| 30 if can_fail_build: | |
| 31 raise | |
| 32 else: | |
| 33 return f.result | |
| 34 | |
| 35 def fetch_tags(self, remote_name=None, **kwargs): | |
| 36 """Fetches all tags from the remote.""" | |
| 37 kwargs.setdefault('name', 'git fetch tags') | |
| 38 remote_name = remote_name or 'origin' | |
| 39 return self('fetch', remote_name, '--tags', **kwargs) | |
| 40 | |
| 41 def cat_file_at_commit(self, file_path, commit_hash, remote_name=None, | |
| 42 **kwargs): | |
| 43 """Outputs the contents of a file at a given revision.""" | |
| 44 self.fetch_tags(remote_name=remote_name, **kwargs) | |
| 45 kwargs.setdefault('name', 'git cat-file %s:%s' % (commit_hash, file_path)) | |
| 46 return self('cat-file', 'blob', '%s:%s' % (commit_hash, file_path), | |
| 47 **kwargs) | |
| 48 | |
| 49 def count_objects(self, previous_result=None, can_fail_build=False, **kwargs): | |
| 50 """Returns `git count-objects` result as a dict. | |
| 51 | |
| 52 Args: | |
| 53 previous_result (dict): the result of previous count_objects call. | |
| 54 If passed, delta is reported in the log and step text. | |
| 55 can_fail_build (bool): if True, may fail the build and/or raise an | |
| 56 exception. Defaults to False. | |
| 57 | |
| 58 Returns: | |
| 59 A dict of count-object values, or None if count-object run failed. | |
| 60 """ | |
| 61 if previous_result: | |
| 62 assert isinstance(previous_result, dict) | |
| 63 assert all(isinstance(v, long) for v in previous_result.itervalues()) | |
| 64 assert 'size' in previous_result | |
| 65 assert 'size-pack' in previous_result | |
| 66 | |
| 67 step_result = None | |
| 68 try: | |
| 69 step_result = self( | |
| 70 'count-objects', '-v', stdout=self.m.raw_io.output(), | |
| 71 can_fail_build=can_fail_build, **kwargs) | |
| 72 | |
| 73 if not step_result.stdout: | |
| 74 return None | |
| 75 | |
| 76 result = {} | |
| 77 for line in step_result.stdout.splitlines(): | |
| 78 name, value = line.split(':', 1) | |
| 79 result[name] = long(value.strip()) | |
| 80 | |
| 81 def results_to_text(results): | |
| 82 return [' %s: %s' % (k, v) for k, v in results.iteritems()] | |
| 83 | |
| 84 step_result.presentation.logs['result'] = results_to_text(result) | |
| 85 | |
| 86 if previous_result: | |
| 87 delta = { | |
| 88 key: value - previous_result[key] | |
| 89 for key, value in result.iteritems() | |
| 90 if key in previous_result} | |
| 91 step_result.presentation.logs['delta'] = ( | |
| 92 ['before:'] + results_to_text(previous_result) + | |
| 93 ['', 'after:'] + results_to_text(result) + | |
| 94 ['', 'delta:'] + results_to_text(delta) | |
| 95 ) | |
| 96 | |
| 97 size_delta = ( | |
| 98 result['size'] + result['size-pack'] | |
| 99 - previous_result['size'] - previous_result['size-pack']) | |
| 100 # size_delta is in KiB. | |
| 101 step_result.presentation.step_text = ( | |
| 102 'size delta: %+.2f MiB' % (size_delta / 1024.0)) | |
| 103 | |
| 104 return result | |
| 105 except Exception as ex: | |
| 106 if step_result: | |
| 107 step_result.presentation.logs['exception'] = ['%r' % ex] | |
| 108 step_result.presentation.status = self.m.step.WARNING | |
| 109 if can_fail_build: | |
| 110 raise recipe_api.InfraFailure('count-objects failed: %s' % ex) | |
| 111 return None | |
| 112 | |
| 113 def checkout(self, url, ref=None, dir_path=None, recursive=False, | |
| 114 submodules=True, submodule_update_force=False, | |
| 115 keep_paths=None, step_suffix=None, | |
| 116 curl_trace_file=None, can_fail_build=True, | |
| 117 set_got_revision=False, remote_name=None, | |
| 118 display_fetch_size=None, file_name=None, | |
| 119 submodule_update_recursive=True): | |
| 120 """Returns an iterable of steps to perform a full git checkout. | |
| 121 Args: | |
| 122 url (str): url of remote repo to use as upstream | |
| 123 ref (str): ref to fetch and check out | |
| 124 dir_path (Path): optional directory to clone into | |
| 125 recursive (bool): whether to recursively fetch submodules or not | |
| 126 submodules (bool): whether to sync and update submodules or not | |
| 127 submodule_update_force (bool): whether to update submodules with --force | |
| 128 keep_paths (iterable of strings): paths to ignore during git-clean; | |
| 129 paths are gitignore-style patterns relative to checkout_path. | |
| 130 step_suffix (str): suffix to add to a each step name | |
| 131 curl_trace_file (Path): if not None, dump GIT_CURL_VERBOSE=1 trace to that | |
| 132 file. Useful for debugging git issue reproducible only on bots. It has | |
| 133 a side effect of all stderr output of 'git fetch' going to that file. | |
| 134 can_fail_build (bool): if False, ignore errors during fetch or checkout. | |
| 135 set_got_revision (bool): if True, resolves HEAD and sets got_revision | |
| 136 property. | |
| 137 remote_name (str): name of the git remote to use | |
| 138 display_fetch_size (bool): if True, run `git count-objects` before and | |
| 139 after fetch and display delta. Adds two more steps. Defaults to False. | |
| 140 file_name (str): optional path to a single file to checkout. | |
| 141 submodule_update_recursive (bool): if True, updates submodules | |
| 142 recursively. | |
| 143 """ | |
| 144 # TODO(robertocn): Break this function and refactor calls to it. | |
| 145 # The problem is that there are way too many unrealated use cases for | |
| 146 # it, and the function's signature is getting unwieldy and its body | |
| 147 # unreadable. | |
| 148 display_fetch_size = display_fetch_size or False | |
| 149 if not dir_path: | |
| 150 dir_path = url.rsplit('/', 1)[-1] | |
| 151 if dir_path.endswith('.git'): # ex: https://host/foobar.git | |
| 152 dir_path = dir_path[:-len('.git')] | |
| 153 | |
| 154 # ex: ssh://host:repo/foobar/.git | |
| 155 dir_path = dir_path or dir_path.rsplit('/', 1)[-1] | |
| 156 | |
| 157 dir_path = self.m.path['slave_build'].join(dir_path) | |
| 158 | |
| 159 if 'checkout' not in self.m.path: | |
| 160 self.m.path['checkout'] = dir_path | |
| 161 | |
| 162 git_setup_args = ['--path', dir_path, '--url', url] | |
| 163 | |
| 164 if remote_name: | |
| 165 git_setup_args += ['--remote', remote_name] | |
| 166 else: | |
| 167 remote_name = 'origin' | |
| 168 | |
| 169 if self.m.platform.is_win: | |
| 170 git_setup_args += ['--git_cmd_path', | |
| 171 self.m.path['depot_tools'].join('git.bat')] | |
| 172 | |
| 173 step_suffix = '' if step_suffix is None else ' (%s)' % step_suffix | |
| 174 self.m.python( | |
| 175 'git setup%s' % step_suffix, | |
| 176 self.resource('git_setup.py'), | |
| 177 git_setup_args) | |
| 178 | |
| 179 # There are five kinds of refs we can be handed: | |
| 180 # 0) None. In this case, we default to properties['branch']. | |
| 181 # 1) A 40-character SHA1 hash. | |
| 182 # 2) A fully-qualifed arbitrary ref, e.g. 'refs/foo/bar/baz'. | |
| 183 # 3) A fully qualified branch name, e.g. 'refs/heads/master'. | |
| 184 # Chop off 'refs/heads' and now it matches case (4). | |
| 185 # 4) A branch name, e.g. 'master'. | |
| 186 # Note that 'FETCH_HEAD' can be many things (and therefore not a valid | |
| 187 # checkout target) if many refs are fetched, but we only explicitly fetch | |
| 188 # one ref here, so this is safe. | |
| 189 fetch_args = [] | |
| 190 if not ref: # Case 0 | |
| 191 fetch_remote = remote_name | |
| 192 fetch_ref = self.m.properties.get('branch') or 'master' | |
| 193 checkout_ref = 'FETCH_HEAD' | |
| 194 elif self._GIT_HASH_RE.match(ref): # Case 1. | |
| 195 fetch_remote = remote_name | |
| 196 fetch_ref = '' | |
| 197 checkout_ref = ref | |
| 198 elif ref.startswith('refs/heads/'): # Case 3. | |
| 199 fetch_remote = remote_name | |
| 200 fetch_ref = ref[len('refs/heads/'):] | |
| 201 checkout_ref = 'FETCH_HEAD' | |
| 202 else: # Cases 2 and 4. | |
| 203 fetch_remote = remote_name | |
| 204 fetch_ref = ref | |
| 205 checkout_ref = 'FETCH_HEAD' | |
| 206 | |
| 207 fetch_args = [x for x in (fetch_remote, fetch_ref) if x] | |
| 208 if recursive: | |
| 209 fetch_args.append('--recurse-submodules') | |
| 210 | |
| 211 fetch_env = {} | |
| 212 fetch_stderr = None | |
| 213 if curl_trace_file: | |
| 214 fetch_env['GIT_CURL_VERBOSE'] = '1' | |
| 215 fetch_stderr = self.m.raw_io.output(leak_to=curl_trace_file) | |
| 216 | |
| 217 fetch_step_name = 'git fetch%s' % step_suffix | |
| 218 if display_fetch_size: | |
| 219 count_objects_before_fetch = self.count_objects( | |
| 220 name='count-objects before %s' % fetch_step_name, | |
| 221 cwd=dir_path, | |
| 222 step_test_data=lambda: self.m.raw_io.test_api.stream_output( | |
| 223 self.test_api.count_objects_output(1000))) | |
| 224 self('retry', 'fetch', *fetch_args, | |
| 225 cwd=dir_path, | |
| 226 name=fetch_step_name, | |
| 227 env=fetch_env, | |
| 228 stderr=fetch_stderr, | |
| 229 can_fail_build=can_fail_build) | |
| 230 if display_fetch_size: | |
| 231 self.count_objects( | |
| 232 name='count-objects after %s' % fetch_step_name, | |
| 233 cwd=dir_path, | |
| 234 previous_result=count_objects_before_fetch, | |
| 235 step_test_data=lambda: self.m.raw_io.test_api.stream_output( | |
| 236 self.test_api.count_objects_output(2000))) | |
| 237 | |
| 238 if file_name: | |
| 239 self('checkout', '-f', checkout_ref, '--', file_name, | |
| 240 cwd=dir_path, | |
| 241 name='git checkout%s' % step_suffix, | |
| 242 can_fail_build=can_fail_build) | |
| 243 | |
| 244 else: | |
| 245 self('checkout', '-f', checkout_ref, | |
| 246 cwd=dir_path, | |
| 247 name='git checkout%s' % step_suffix, | |
| 248 can_fail_build=can_fail_build) | |
| 249 | |
| 250 if set_got_revision: | |
| 251 rev_parse_step = self('rev-parse', 'HEAD', | |
| 252 cwd=dir_path, | |
| 253 name='set got_revision', | |
| 254 stdout=self.m.raw_io.output(), | |
| 255 can_fail_build=False) | |
| 256 | |
| 257 if rev_parse_step.presentation.status == 'SUCCESS': | |
| 258 sha = rev_parse_step.stdout.strip() | |
| 259 rev_parse_step.presentation.properties['got_revision'] = sha | |
| 260 | |
| 261 clean_args = list(itertools.chain( | |
| 262 *[('-e', path) for path in keep_paths or []])) | |
| 263 | |
| 264 self('clean', '-f', '-d', '-x', *clean_args, | |
| 265 name='git clean%s' % step_suffix, | |
| 266 cwd=dir_path, | |
| 267 can_fail_build=can_fail_build) | |
| 268 | |
| 269 if submodules: | |
| 270 self('submodule', 'sync', | |
| 271 name='submodule sync%s' % step_suffix, | |
| 272 cwd=dir_path, | |
| 273 can_fail_build=can_fail_build) | |
| 274 submodule_update = ['submodule', 'update', '--init'] | |
| 275 if submodule_update_recursive: | |
| 276 submodule_update.append('--recursive') | |
| 277 if submodule_update_force: | |
| 278 submodule_update.append('--force') | |
| 279 self(*submodule_update, | |
| 280 name='submodule update%s' % step_suffix, | |
| 281 cwd=dir_path, | |
| 282 can_fail_build=can_fail_build) | |
| 283 | |
| 284 def get_timestamp(self, commit='HEAD', test_data=None, **kwargs): | |
| 285 """Find and return the timestamp of the given commit.""" | |
| 286 step_test_data = None | |
| 287 if test_data is not None: | |
| 288 step_test_data = lambda: self.m.raw_io.test_api.stream_output(test_data) | |
| 289 return self('show', commit, '--format=%at', '-s', | |
| 290 stdout=self.m.raw_io.output(), | |
| 291 step_test_data=step_test_data).stdout.rstrip() | |
| 292 | |
| 293 def rebase(self, name_prefix, branch, dir_path, remote_name=None, | |
| 294 **kwargs): | |
| 295 """Run rebase HEAD onto branch | |
| 296 Args: | |
| 297 name_prefix (str): a prefix used for the step names | |
| 298 branch (str): a branch name or a hash to rebase onto | |
| 299 dir_path (Path): directory to clone into | |
| 300 remote_name (str): the remote name to rebase from if not origin | |
| 301 """ | |
| 302 remote_name = remote_name or 'origin' | |
| 303 try: | |
| 304 self('rebase', '%s/master' % remote_name, | |
| 305 name="%s rebase" % name_prefix, cwd=dir_path, **kwargs) | |
| 306 except self.m.step.StepFailure: | |
| 307 self('rebase', '--abort', name='%s rebase abort' % name_prefix, | |
| 308 cwd=dir_path, **kwargs) | |
| 309 raise | |
| 310 | |
| 311 def config_get(self, prop_name, **kwargs): | |
| 312 """Returns: (str) The Git config output, or None if no output was generated. | |
| 313 | |
| 314 Args: | |
| 315 prop_name: (str) The name of the config property to query. | |
| 316 kwargs: Forwarded to '__call__'. | |
| 317 """ | |
| 318 kwargs['name'] = kwargs.get('name', 'git config %s' % (prop_name,)) | |
| 319 result = self('config', '--get', prop_name, stdout=self.m.raw_io.output(), | |
| 320 **kwargs) | |
| 321 | |
| 322 value = result.stdout | |
| 323 if value: | |
| 324 value = value.strip() | |
| 325 result.presentation.step_text = value | |
| 326 return value | |
| 327 | |
| 328 def get_remote_url(self, remote_name=None, **kwargs): | |
| 329 """Returns: (str) The URL of the remote Git repository, or None. | |
| 330 | |
| 331 Args: | |
| 332 remote_name: (str) The name of the remote to query, defaults to 'origin'. | |
| 333 kwargs: Forwarded to '__call__'. | |
| 334 """ | |
| 335 remote_name = remote_name or 'origin' | |
| 336 return self.config_get('remote.%s.url' % (remote_name,), **kwargs) | |
| 337 | |
| 338 def bundle_create(self, bundle_path, rev_list_args=None, **kwargs): | |
| 339 """Run 'git bundle create' on a Git repository. | |
| 340 | |
| 341 Args: | |
| 342 bundle_path (Path): The path of the output bundle. | |
| 343 refs (list): The list of refs to include in the bundle. If None, all | |
| 344 refs in the Git checkout will be bundled. | |
| 345 kwargs: Forwarded to '__call__'. | |
| 346 """ | |
| 347 if not rev_list_args: | |
| 348 rev_list_args = ['--all'] | |
| 349 self('bundle', 'create', bundle_path, *rev_list_args, **kwargs) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1644173002:
Revert of depot_tools: import bot_update gclient git rietveld tryserver recipe modules (Closed)
Base URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git@master