OLD | NEW |
1 # Copyright 2016 The LUCI Authors. All rights reserved. | 1 # Copyright 2016 The LUCI Authors. All rights reserved. |
2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
4 | 4 |
5 import calendar | 5 import calendar |
6 import httplib | 6 import httplib |
7 import json | 7 import json |
8 import logging | 8 import logging |
9 import os | 9 import os |
10 import re | 10 import re |
(...skipping 13 matching lines...) Expand all Loading... |
24 from . import env | 24 from . import env |
25 from . import requests_ssl | 25 from . import requests_ssl |
26 from .requests_ssl import requests | 26 from .requests_ssl import requests |
27 | 27 |
28 import subprocess42 | 28 import subprocess42 |
29 from google.protobuf import json_format | 29 from google.protobuf import json_format |
30 | 30 |
31 LOGGER = logging.getLogger(__name__) | 31 LOGGER = logging.getLogger(__name__) |
32 | 32 |
33 | 33 |
34 def has_interesting_changes(spec, changed_files): | |
35 # TODO(iannucci): analyze bundle_extra_paths.txt too. | |
36 return ( | |
37 'infra/config/recipes.cfg' in changed_files or | |
38 any(f.startswith(spec.recipes_path) for f in changed_files) | |
39 ) | |
40 | |
41 | |
42 class FetchError(Exception): | 34 class FetchError(Exception): |
43 pass | 35 pass |
44 | 36 |
45 | 37 |
46 class FetchNotAllowedError(FetchError): | 38 class FetchNotAllowedError(FetchError): |
47 pass | 39 pass |
48 | 40 |
49 | 41 |
50 class UnresolvedRefspec(Exception): | 42 class UnresolvedRefspec(Exception): |
51 pass | 43 pass |
52 | 44 |
53 | 45 |
54 # revision (str): the revision of this commit (i.e. hash) | 46 # revision (str): the revision of this commit (i.e. hash) |
55 # author_email (str|None): the email of the author of this commit | 47 # author_email (str|None): the email of the author of this commit |
56 # commit_timestamp (int): the unix commit timestamp for this commit | 48 # commit_timestamp (int): the unix commit timestamp for this commit |
57 # message_lines (tuple(str)): the message of this commit | 49 # message_lines (tuple(str)): the message of this commit |
58 # spec (package_pb2.Package): the parsed infra/config/recipes.cfg file or None. | 50 # spec (package_pb2.Package): the parsed infra/config/recipes.cfg file or None. |
59 # roll_candidate (bool): if this commit contains changes which are known to | |
60 # affect the behavior of the recipes (i.e. modifications within recipe_path | |
61 # and/or modifications to recipes.cfg) | |
62 CommitMetadata = namedtuple( | 51 CommitMetadata = namedtuple( |
63 '_CommitMetadata', | 52 '_CommitMetadata', |
64 'revision author_email commit_timestamp message_lines spec roll_candidate') | 53 'revision author_email commit_timestamp message_lines spec') |
65 | 54 |
66 | 55 |
67 class Backend(object): | 56 class Backend(object): |
68 @staticmethod | 57 @staticmethod |
69 def class_for_type(repo_type): | 58 def class_for_type(repo_type): |
70 """ | 59 """ |
71 Args: | 60 Args: |
72 repo_type (package_pb2.DepSpec.RepoType) | 61 repo_type (package_pb2.DepSpec.RepoType) |
73 | 62 |
74 Returns Backend (class): Returns the Backend appropriate for the | 63 Returns Backend (class): Returns the Backend appropriate for the |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
118 _GIT_METADATA_CACHE = {} | 107 _GIT_METADATA_CACHE = {} |
119 | 108 |
120 # This matches git commit hashes. | 109 # This matches git commit hashes. |
121 _COMMIT_RE = re.compile(r'^[a-fA-F0-9]{40}$') | 110 _COMMIT_RE = re.compile(r'^[a-fA-F0-9]{40}$') |
122 | 111 |
123 def commit_metadata(self, refspec): | 112 def commit_metadata(self, refspec): |
124 """Cached version of _commit_metadata_impl. | 113 """Cached version of _commit_metadata_impl. |
125 | 114 |
126 The refspec will be resolved if it's not absolute. | 115 The refspec will be resolved if it's not absolute. |
127 | 116 |
128 Returns (CommitMetadata). | 117 Returns { |
| 118 'author': '<author name>', |
| 119 'message': '<commit message>', |
| 120 'spec': package_pb2.Package or None, # the parsed recipes.cfg file. |
| 121 } |
129 """ | 122 """ |
130 revision = self.resolve_refspec(refspec) | 123 revision = self.resolve_refspec(refspec) |
131 cache = self._GIT_METADATA_CACHE.setdefault(self.repo_url, {}) | 124 cache = self._GIT_METADATA_CACHE.setdefault(self.repo_url, {}) |
132 if revision not in cache: | 125 if revision not in cache: |
133 cache[revision] = self._commit_metadata_impl(revision) | 126 cache[revision] = self._commit_metadata_impl(revision) |
134 return cache[revision] | 127 return cache[revision] |
135 | 128 |
136 @classmethod | 129 @classmethod |
137 def is_resolved_revision(cls, revision): | 130 def is_resolved_revision(cls, revision): |
138 return cls._COMMIT_RE.match(revision) | 131 return cls._COMMIT_RE.match(revision) |
139 | 132 |
140 @classmethod | 133 @classmethod |
141 def assert_resolved(cls, revision): | 134 def assert_resolved(cls, revision): |
142 if not cls.is_resolved_revision(revision): | 135 if not cls.is_resolved_revision(revision): |
143 raise UnresolvedRefspec('unresolved refspec %r' % revision) | 136 raise UnresolvedRefspec('unresolved refspec %r' % revision) |
144 | 137 |
145 def resolve_refspec(self, refspec): | 138 def resolve_refspec(self, refspec): |
146 if self.is_resolved_revision(refspec): | 139 if self.is_resolved_revision(refspec): |
147 return refspec | 140 return refspec |
148 return self._resolve_refspec_impl(refspec) | 141 return self._resolve_refspec_impl(refspec) |
149 | 142 |
150 def updates(self, revision, other_revision): | 143 def updates(self, revision, other_revision, paths): |
151 """Returns a list of revisions |revision| through |other_revision| | 144 """Returns a list of revisions |revision| through |other_revision| |
152 (inclusive). | 145 (inclusive). |
153 | 146 |
| 147 If |paths| is a non-empty list, the history is scoped just to these paths. |
| 148 |
154 Returns list(CommitMetadata) - The commit metadata in the range | 149 Returns list(CommitMetadata) - The commit metadata in the range |
155 (revision,other_revision]. | 150 (revision,other_revision]. |
156 """ | 151 """ |
157 self.assert_resolved(revision) | 152 self.assert_resolved(revision) |
158 self.assert_resolved(other_revision) | 153 self.assert_resolved(other_revision) |
159 return self._updates_impl(revision, other_revision) | 154 return self._updates_impl(revision, other_revision, paths) |
160 | 155 |
161 ### direct overrides. These are public methods which must be overridden. | 156 ### direct overrides. These are public methods which must be overridden. |
162 | 157 |
163 @property | 158 @property |
164 def repo_type(self): | 159 def repo_type(self): |
165 """Returns package_pb2.DepSpec.RepoType.""" | 160 """Returns package_pb2.DepSpec.RepoType.""" |
166 raise NotImplementedError() | 161 raise NotImplementedError() |
167 | 162 |
168 def fetch(self, refspec): | 163 def fetch(self, refspec): |
169 """Does a fetch for the provided refspec (e.g. get all data from remote), if | 164 """Does a fetch for the provided refspec (e.g. get all data from remote), if |
(...skipping 13 matching lines...) Expand all Loading... |
183 remote git repo, e.g. 'refs/heads/master', 'deadbeef...face', etc. | 178 remote git repo, e.g. 'refs/heads/master', 'deadbeef...face', etc. |
184 """ | 179 """ |
185 # TODO(iannucci): Alter the contract for this method so that it only checks | 180 # TODO(iannucci): Alter the contract for this method so that it only checks |
186 # out the files referred to according to the rules that the bundle | 181 # out the files referred to according to the rules that the bundle |
187 # subcommand uses. | 182 # subcommand uses. |
188 raise NotImplementedError() | 183 raise NotImplementedError() |
189 | 184 |
190 ### private overrides. Override these in the implementations, but don't call | 185 ### private overrides. Override these in the implementations, but don't call |
191 ### externally. | 186 ### externally. |
192 | 187 |
193 def _updates_impl(self, revision, other_revision): | 188 def _updates_impl(self, revision, other_revision, paths): |
194 """Returns a list of revisions |revision| through |other_revision|. This | 189 """Returns a list of revisions |revision| through |other_revision|. This |
195 includes |revision| and |other_revision|. | 190 includes |revision| and |other_revision|. |
196 | 191 |
| 192 If |paths| is a non-empty list, the history is scoped just to these paths. |
| 193 |
197 Args: | 194 Args: |
198 revision (str) - the first git commit | 195 revision (str) - the first git commit |
199 other_revision (str) - the second git commit | 196 other_revision (str) - the second git commit |
200 | 197 |
201 Returns list(CommitMetadata) - The commit metadata in the range | 198 Returns list(CommitMetadata) - The commit metadata in the range |
202 [revision,other_revision]. | 199 [revision,other_revision]. |
203 """ | 200 """ |
204 raise NotImplementedError() | 201 raise NotImplementedError() |
205 | 202 |
206 def _resolve_refspec_impl(self, refspec): | 203 def _resolve_refspec_impl(self, refspec): |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
241 def _git(self, *args): | 238 def _git(self, *args): |
242 """Runs a git command. | 239 """Runs a git command. |
243 | 240 |
244 Will automatically set low speed limit/time, and cd into the checkout_dir. | 241 Will automatically set low speed limit/time, and cd into the checkout_dir. |
245 | 242 |
246 Args: | 243 Args: |
247 *args (str) - The list of command arguments to pass to git. | 244 *args (str) - The list of command arguments to pass to git. |
248 | 245 |
249 Raises GitError on failure. | 246 Raises GitError on failure. |
250 """ | 247 """ |
251 if self._GIT_BINARY.endswith('.bat'): | |
252 # On the esteemed Windows Operating System, '^' is an escape character. | |
253 # Since .bat files are running cmd.exe under the hood, they interpret this | |
254 # escape character. We need to ultimately get a single ^, so we need two | |
255 # ^'s for when we invoke the .bat, and each of those needs to be escaped | |
256 # when the bat ultimately invokes the git.exe binary. This leaves us with | |
257 # a total of 4x the ^'s that we originally wanted. Hooray. | |
258 args = [a.replace('^', '^^^^') for a in args] | |
259 | |
260 cmd = [ | 248 cmd = [ |
261 self._GIT_BINARY, | 249 self._GIT_BINARY, |
262 '-C', self.checkout_dir, | 250 '-C', self.checkout_dir, |
263 ] + list(args) | 251 ] + list(args) |
264 | 252 |
265 try: | 253 try: |
266 return self._execute(*cmd) | 254 return self._execute(*cmd) |
267 except subprocess42.CalledProcessError as e: | 255 except subprocess42.CalledProcessError as e: |
268 subcommand = (args[0]) if args else ('') | 256 subcommand = (args[0]) if args else ('') |
269 raise GitError('Git "%s" failed: %s' % (subcommand, e.message,)) | 257 raise GitError('Git "%s" failed: %s' % (subcommand, e.message,)) |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
332 self.fetch(refspec) | 320 self.fetch(refspec) |
333 | 321 |
334 # reset touches index.lock which is problematic when multiple processes are | 322 # reset touches index.lock which is problematic when multiple processes are |
335 # accessing the recipes at the same time. To allieviate this, we do a quick | 323 # accessing the recipes at the same time. To allieviate this, we do a quick |
336 # diff, which will exit if `revision` is not already checked out. | 324 # diff, which will exit if `revision` is not already checked out. |
337 try: | 325 try: |
338 self._git('diff', '--quiet', revision) | 326 self._git('diff', '--quiet', revision) |
339 except GitError: | 327 except GitError: |
340 self._git('reset', '-q', '--hard', revision) | 328 self._git('reset', '-q', '--hard', revision) |
341 | 329 |
342 def _updates_impl(self, revision, other_revision): | 330 def _updates_impl(self, revision, other_revision, paths): |
343 args = [ | 331 args = [ |
344 'rev-list', | 332 'rev-list', |
345 '--reverse', | 333 '--reverse', |
346 '--topo-order', | 334 '--topo-order', |
347 '%s..%s' % (revision, other_revision), | 335 '%s..%s' % (revision, other_revision), |
348 ] | 336 ] |
| 337 if paths: |
| 338 args.extend(['--'] + paths) |
349 return [ | 339 return [ |
350 self.commit_metadata(rev) | 340 self.commit_metadata(rev) |
351 for rev in self._git(*args).strip().split('\n') | 341 for rev in self._git(*args).strip().split('\n') |
352 if bool(rev) | 342 if bool(rev) |
353 ] | 343 ] |
354 | 344 |
355 def _resolve_refspec_impl(self, revision): | 345 def _resolve_refspec_impl(self, revision): |
356 self._ensure_local_repo_exists() | 346 self._ensure_local_repo_exists() |
357 self.assert_remote('resolve refspec %r' % revision) | 347 self.assert_remote('resolve refspec %r' % revision) |
358 rslt = self._git('ls-remote', self.repo_url, revision).split()[0] | 348 rslt = self._git('ls-remote', self.repo_url, revision).split()[0] |
(...skipping 11 matching lines...) Expand all Loading... |
370 # %`Body` | 360 # %`Body` |
371 meta = self._git( | 361 meta = self._git( |
372 'show', '-s', '--format=%aE%n%ct%n%B', revision).rstrip('\n').splitlines() | 362 'show', '-s', '--format=%aE%n%ct%n%B', revision).rstrip('\n').splitlines() |
373 | 363 |
374 try: | 364 try: |
375 spec = package_io.parse(self._git( | 365 spec = package_io.parse(self._git( |
376 'cat-file', 'blob', '%s:infra/config/recipes.cfg' % revision)) | 366 'cat-file', 'blob', '%s:infra/config/recipes.cfg' % revision)) |
377 except GitError: | 367 except GitError: |
378 spec = None | 368 spec = None |
379 | 369 |
380 # check diff to see if it touches anything interesting. | |
381 changed_files = set(self._git( | |
382 'diff-tree', '-r', '--no-commit-id', '--name-only', '%s^!' % revision) | |
383 .splitlines()) | |
384 | |
385 return CommitMetadata(revision, meta[0], | 370 return CommitMetadata(revision, meta[0], |
386 int(meta[1]), tuple(meta[2:]), | 371 int(meta[1]), tuple(meta[2:]), |
387 spec, has_interesting_changes(spec, changed_files)) | 372 spec) |
388 | 373 |
389 class GitilesFetchError(FetchError): | 374 class GitilesFetchError(FetchError): |
390 """An HTTP error that occurred during Gitiles fetching.""" | 375 """An HTTP error that occurred during Gitiles fetching.""" |
391 | 376 |
392 def __init__(self, status, message): | 377 def __init__(self, status, message): |
393 super(GitilesFetchError, self).__init__( | 378 super(GitilesFetchError, self).__init__( |
394 'Gitiles error code (%d): %s' % (status, message)) | 379 'Gitiles error code (%d): %s' % (status, message)) |
395 self.status = status | 380 self.status = status |
396 self.message = message | 381 self.message = message |
397 | 382 |
398 @staticmethod | 383 @staticmethod |
399 def transient(e): | 384 def transient(e): |
400 """ | 385 """ |
401 Returns (bool): True if "e" is a GitilesFetchError with transient HTTP code. | 386 Returns (bool): True if "e" is a GitilesFetchError with transient HTTP code. |
402 """ | 387 """ |
403 return (isinstance(e, GitilesFetchError) and | 388 return (isinstance(e, GitilesFetchError) and |
404 e.status >= httplib.INTERNAL_SERVER_ERROR) | 389 e.status >= httplib.INTERNAL_SERVER_ERROR) |
405 | 390 |
406 | 391 |
407 # Internal cache object for GitilesBackend. | 392 # Internal cache object for GitilesBackend. |
408 # commit (str) - the git commit hash | 393 # commit (str) - the git commit hash |
409 # author_email (str) - the author email for this commit | 394 # author_email (str) - the author email for this commit |
410 # message_lines (tuple) - the lines of the commit message | 395 # message_lines (tuple) - the lines of the commit message |
411 # changed_files (frozenset) - all paths touched by this commit | |
412 class _GitilesCommitJson(namedtuple( | 396 class _GitilesCommitJson(namedtuple( |
413 '_GitilesCommitJson', | 397 '_GitilesCommitJson', |
414 'commit author_email commit_timestamp message_lines changed_files')): | 398 'commit author_email commit_timestamp message_lines')): |
415 @classmethod | 399 @classmethod |
416 def from_raw_json(cls, raw): | 400 def from_raw_json(cls, raw): |
417 mod_files = set() | |
418 for entry in raw['tree_diff']: | |
419 mod_files.add(entry['old_path']) | |
420 mod_files.add(entry['new_path']) | |
421 return cls( | 401 return cls( |
422 raw['commit'], | 402 raw['commit'], |
423 raw['author']['email'], | 403 raw['author']['email'], |
424 calendar.timegm(time.strptime(raw['committer']['time'])), | 404 calendar.timegm(time.strptime(raw['committer']['time'])), |
425 tuple(raw['message'].splitlines()), | 405 tuple(raw['message'].splitlines()), |
426 frozenset(mod_files), | |
427 ) | 406 ) |
428 | 407 |
429 | 408 |
430 class GitilesBackend(Backend): | 409 class GitilesBackend(Backend): |
431 """GitilesBackend uses a repo served by Gitiles.""" | 410 """GitilesBackend uses a repo served by Gitiles.""" |
432 | 411 |
433 # Prefix at the beginning of Gerrit/Gitiles JSON API responses. | 412 # Prefix at the beginning of Gerrit/Gitiles JSON API responses. |
434 _GERRIT_XSRF_HEADER = ')]}\'\n' | 413 _GERRIT_XSRF_HEADER = ')]}\'\n' |
435 | 414 |
436 @util.exponential_retry(condition=GitilesFetchError.transient) | 415 @util.exponential_retry(condition=GitilesFetchError.transient) |
437 def _fetch_gitiles(self, url_fmt, *args): | 416 def _fetch_gitiles(self, url_fmt, *args): |
438 """Fetches a remote URL path and returns the response object on success. | 417 """Fetches a remote URL path and returns the response object on success. |
439 | 418 |
440 Args: | 419 Args: |
441 url_fmt (str) - the url path fragment as a python %format string, like | 420 url_fmt (str) - the url path fragment as a python %format string, like |
442 '%s/foo/bar?something=value' | 421 '%s/foo/bar?something=value' |
443 *args (str) - the arguments to format url_fmt with. They will be URL | 422 *args (str) - the arguments to format url_fmt with. They will be URL |
444 escaped. | 423 escaped. |
445 | 424 |
446 Returns requests.Response. | 425 Returns requests.Response. |
447 """ | 426 """ |
448 url = '%s/%s' % (self.repo_url, | 427 url = '%s/%s' % (self.repo_url, |
449 url_fmt % tuple(map(requests.utils.quote, args))) | 428 url_fmt % tuple(map(requests.utils.quote, args))) |
450 LOGGER.info('fetching %s' % url) | 429 LOGGER.info('fetching %s' % url) |
451 resp = requests.get(url) | 430 resp = requests.get(url) |
452 if resp.status_code != httplib.OK: | 431 if resp.status_code != httplib.OK: |
453 raise GitilesFetchError(resp.status_code, resp.text) | 432 raise GitilesFetchError(resp.status_code, resp.text) |
454 return resp | 433 return resp |
455 | 434 |
456 def _fetch_gitiles_committish_json(self, url_fmt, *args): | 435 def _fetch_gitiles_json(self, url_fmt, *args): |
457 """Fetches a remote URL path and expects a JSON object on success. | 436 """Fetches a remote URL path and expects a JSON object on success. |
458 | 437 |
459 This appends two GET params to url_fmt: | |
460 format=JSON - Does what you expect | |
461 name-status=1 - Ensures that commit objects returned have a 'tree_diff' | |
462 member which shows the diff for that commit. | |
463 | |
464 Args: | 438 Args: |
465 url_fmt (str) - the url path fragment as a python %format string, like | 439 url_fmt (str) - the url path fragment as a python %format string, like |
466 '%s/foo/bar?something=value' | 440 '%s/foo/bar?something=value' |
467 *args (str) - the arguments to format url_fmt with. They will be URL | 441 *args (str) - the arguments to format url_fmt with. They will be URL |
468 escaped. | 442 escaped. |
469 | 443 |
470 Returns the decoded JSON object | 444 Returns the decoded JSON object |
471 """ | 445 """ |
472 resp = self._fetch_gitiles(url_fmt+'?name-status=1&format=JSON', *args) | 446 resp = self._fetch_gitiles(url_fmt, *args) |
473 if not resp.text.startswith(self._GERRIT_XSRF_HEADER): | 447 if not resp.text.startswith(self._GERRIT_XSRF_HEADER): |
474 raise GitilesFetchError(resp.status_code, 'Missing XSRF prefix') | 448 raise GitilesFetchError(resp.status_code, 'Missing XSRF prefix') |
475 return json.loads(resp.text[len(self._GERRIT_XSRF_HEADER):]) | 449 return json.loads(resp.text[len(self._GERRIT_XSRF_HEADER):]) |
476 | 450 |
477 # This caches entries from _fetch_commit_json. It's populated by | 451 # This caches entries from _fetch_commit_json. It's populated by |
478 # _fetch_commit_json as well as _updates_impl. | 452 # _fetch_commit_json as well as _updates_impl. |
479 # | 453 # |
480 # Mapping of: | 454 # Mapping of: |
481 # repo_url -> git_revision -> _GitilesCommitJson | 455 # repo_url -> git_revision -> _GitilesCommitJson |
482 # | 456 # |
483 # Only populated if _fetch_commit_json is passed a resolved commit. | 457 # Only populated if _fetch_commit_json is passed a resolved commit. |
484 _COMMIT_JSON_CACHE = {} | 458 _COMMIT_JSON_CACHE = {} |
485 | 459 |
486 def _fetch_commit_json(self, refspec): | 460 def _fetch_commit_json(self, refspec): |
487 """Returns _GitilesCommitJson for the refspec. | 461 """Returns _GitilesCommitJson for the refspec. |
488 | 462 |
489 If refspec is resolved then this value is cached. | 463 If refspec is resolved then this value is cached. |
490 """ | 464 """ |
491 c = self._COMMIT_JSON_CACHE.setdefault(self.repo_url, {}) | 465 c = self._COMMIT_JSON_CACHE.setdefault(self.repo_url, {}) |
492 if refspec in c: | 466 if refspec in c: |
493 return c[refspec] | 467 return c[refspec] |
494 | 468 |
495 raw = self._fetch_gitiles_committish_json('+/%s', refspec) | 469 raw = self._fetch_gitiles_json('+/%s?format=JSON', refspec) |
496 ret = _GitilesCommitJson.from_raw_json(raw) | 470 ret = _GitilesCommitJson.from_raw_json(raw) |
497 if self.is_resolved_revision(refspec): | 471 if self.is_resolved_revision(refspec): |
498 c[refspec] = ret | 472 c[refspec] = ret |
499 | 473 |
500 return ret | 474 return ret |
501 | 475 |
502 | 476 |
503 ### Backend implementations | 477 ### Backend implementations |
504 | 478 |
505 | 479 |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
544 | 518 |
545 # TODO(iannucci): This implementation may be slow if we need to retieve | 519 # TODO(iannucci): This implementation may be slow if we need to retieve |
546 # multiple files/archives from the remote server. Should possibly consider | 520 # multiple files/archives from the remote server. Should possibly consider |
547 # using a thread pool here. | 521 # using a thread pool here. |
548 | 522 |
549 archive_response = self._fetch_gitiles( | 523 archive_response = self._fetch_gitiles( |
550 '+archive/%s/%s.tar.gz', revision, recipes_path_rel) | 524 '+archive/%s/%s.tar.gz', revision, recipes_path_rel) |
551 with tarfile.open(fileobj=StringIO(archive_response.content)) as tf: | 525 with tarfile.open(fileobj=StringIO(archive_response.content)) as tf: |
552 tf.extractall(recipes_path) | 526 tf.extractall(recipes_path) |
553 | 527 |
554 def _updates_impl(self, revision, other_revision): | 528 def _updates_impl(self, revision, other_revision, paths): |
555 self.assert_remote('_updates_impl') | 529 self.assert_remote('_updates_impl') |
556 | 530 |
557 # TODO(iannucci): implement paging | 531 # TODO(iannucci): implement paging |
558 | 532 |
559 log_json = self._fetch_gitiles_committish_json( | 533 # To include info about touched paths (tree_diff), pass name-status=1 below. |
560 '+log/%s..%s', revision, other_revision) | 534 log_json = self._fetch_gitiles_json( |
| 535 '+log/%s..%s?name-status=1&format=JSON', revision, other_revision) |
561 | 536 |
562 c = self._COMMIT_JSON_CACHE.setdefault(self.repo_url, {}) | 537 c = self._COMMIT_JSON_CACHE.setdefault(self.repo_url, {}) |
563 | 538 |
564 results = [] | 539 results = [] |
565 for entry in log_json['log']: | 540 for entry in log_json['log']: |
566 commit = entry['commit'] | 541 commit = entry['commit'] |
567 c[commit] = _GitilesCommitJson.from_raw_json(entry) | 542 c[commit] = _GitilesCommitJson.from_raw_json(entry) |
568 results.append(commit) | 543 |
| 544 matched = False |
| 545 for path in paths: |
| 546 for diff_entry in entry['tree_diff']: |
| 547 if (diff_entry['old_path'].startswith(path) or |
| 548 diff_entry['new_path'].startswith(path)): |
| 549 matched = True |
| 550 break |
| 551 if matched: |
| 552 break |
| 553 if matched or not paths: |
| 554 results.append(commit) |
569 | 555 |
570 results.reverse() | 556 results.reverse() |
571 return map(self.commit_metadata, results) | 557 return map(self.commit_metadata, results) |
572 | 558 |
573 def _resolve_refspec_impl(self, refspec): | 559 def _resolve_refspec_impl(self, refspec): |
574 if self.is_resolved_revision(refspec): | 560 if self.is_resolved_revision(refspec): |
575 return self.commit_metadata(refspec).commit | 561 return self.commit_metadata(refspec).commit |
576 return self._fetch_commit_json(refspec).commit | 562 return self._fetch_commit_json(refspec).commit |
577 | 563 |
578 def _commit_metadata_impl(self, revision): | 564 def _commit_metadata_impl(self, revision): |
579 self.assert_remote('_commit_metadata_impl') | 565 self.assert_remote('_commit_metadata_impl') |
580 rev_json = self._fetch_commit_json(revision) | 566 rev_json = self._fetch_commit_json(revision) |
581 | 567 |
582 recipes_cfg_text = self._fetch_gitiles( | 568 recipes_cfg_text = self._fetch_gitiles( |
583 '+/%s/infra/config/recipes.cfg?format=TEXT', revision | 569 '+/%s/infra/config/recipes.cfg?format=TEXT', revision |
584 ).text.decode('base64') | 570 ).text.decode('base64') |
585 spec = json_format.Parse( | 571 spec = json_format.Parse( |
586 recipes_cfg_text, package_pb2.Package(), ignore_unknown_fields=True) | 572 recipes_cfg_text, package_pb2.Package(), ignore_unknown_fields=True) |
587 | 573 |
588 return CommitMetadata( | 574 return CommitMetadata( |
589 revision, | 575 revision, |
590 rev_json.author_email, | 576 rev_json.author_email, |
591 rev_json.commit_timestamp, | 577 rev_json.commit_timestamp, |
592 rev_json.message_lines, | 578 rev_json.message_lines, |
593 spec, | 579 spec) |
594 has_interesting_changes(spec, rev_json.changed_files)) | |
OLD | NEW |