| OLD | NEW |
| 1 # coding=utf8 | 1 # coding=utf8 |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 """Commit queue manager class. | 5 """Commit queue manager class. |
| 6 | 6 |
| 7 Security implications: | 7 Security implications: |
| 8 | 8 |
| 9 The following hypothesis are made: | 9 The following hypothesis are made: |
| 10 - Commit queue: | 10 - Commit queue: |
| (...skipping 179 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 190 | 190 |
| 191 raise | 191 raise |
| 192 | 192 |
| 193 # If there is an issue in processed_issues that is not in new_issues, | 193 # If there is an issue in processed_issues that is not in new_issues, |
| 194 # discard it. | 194 # discard it. |
| 195 for pending in self.queue.iterate(): | 195 for pending in self.queue.iterate(): |
| 196 # Note that pending.issue is a int but self.queue.pending_commits keys | 196 # Note that pending.issue is a int but self.queue.pending_commits keys |
| 197 # are str due to json support. | 197 # are str due to json support. |
| 198 if pending.issue not in new_issues: | 198 if pending.issue not in new_issues: |
| 199 logging.info('Flushing issue %d' % pending.issue) | 199 logging.info('Flushing issue %d' % pending.issue) |
| 200 self.context.status.send( |
| 201 pending, |
| 202 { 'verification': 'abort', |
| 203 'payload': { |
| 204 'output': 'CQ bit was unchecked on CL. Ignoring.' }}) |
| 200 pending.get_state = lambda: base.IGNORED | 205 pending.get_state = lambda: base.IGNORED |
| 201 self._discard_pending(pending, 'CQ bit was unchecked on CL. Ignoring.') | 206 self._discard_pending(pending, None) |
| 202 | 207 |
| 203 # Find new issues. | 208 # Find new issues. |
| 204 for issue_id in new_issues: | 209 for issue_id in new_issues: |
| 205 if str(issue_id) not in self.queue.pending_commits: | 210 if str(issue_id) not in self.queue.pending_commits: |
| 206 try: | 211 try: |
| 207 issue_data = self.context.rietveld.get_issue_properties( | 212 issue_data = self.context.rietveld.get_issue_properties( |
| 208 issue_id, True) | 213 issue_id, True) |
| 209 except urllib2.HTTPError as e: | 214 except urllib2.HTTPError as e: |
| 210 if e.code in (500, 502, 503): | 215 if e.code in (500, 502, 503): |
| 211 # Temporary AppEngine hiccup. Just log it and continue. | 216 # Temporary AppEngine hiccup. Just log it and continue. |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 259 # Take in account the case where a verifier was removed. | 264 # Take in account the case where a verifier was removed. |
| 260 done = set(pending.verifications.keys()) | 265 done = set(pending.verifications.keys()) |
| 261 missing = expected - done | 266 missing = expected - done |
| 262 if (not missing or pending.get_state() != base.PROCESSING): | 267 if (not missing or pending.get_state() != base.PROCESSING): |
| 263 continue | 268 continue |
| 264 logging.info( | 269 logging.info( |
| 265 'Processing issue %s (%s, %d)' % ( | 270 'Processing issue %s (%s, %d)' % ( |
| 266 pending.issue, missing, pending.get_state())) | 271 pending.issue, missing, pending.get_state())) |
| 267 self._verify_pending(pending) | 272 self._verify_pending(pending) |
| 268 except base.DiscardPending as e: | 273 except base.DiscardPending as e: |
| 269 message = e.status | 274 self._discard_pending(e.pending, e.status) |
| 270 if not message: | |
| 271 message = 'process_new_pending_commit: ' + self.FAILED_NO_MESSAGE | |
| 272 self._discard_pending(e.pending, message) | |
| 273 | 275 |
| 274 def update_status(self): | 276 def update_status(self): |
| 275 """Updates the status for each pending commit verifier.""" | 277 """Updates the status for each pending commit verifier.""" |
| 276 why_nots = dict((p.issue, p.why_not()) for p in self.queue.iterate()) | 278 why_nots = dict((p.issue, p.why_not()) for p in self.queue.iterate()) |
| 277 | 279 |
| 278 for verifier in self.all_verifiers: | 280 for verifier in self.all_verifiers: |
| 279 try: | 281 try: |
| 280 verifier.update_status(self.queue.iterate()) | 282 verifier.update_status(self.queue.iterate()) |
| 281 except base.DiscardPending as e: | 283 except base.DiscardPending as e: |
| 282 # It's not efficient since it takes a full loop for each pending | 284 # It's not efficient since it takes a full loop for each pending |
| 283 # commit to discard. | 285 # commit to discard. |
| 284 message = e.status | 286 self._discard_pending(e.pending, e.status) |
| 285 if not message: | |
| 286 message = 'update_status: ' + self.FAILED_NO_MESSAGE | |
| 287 self._discard_pending(e.pending, message) | |
| 288 | 287 |
| 289 for pending in self.queue.iterate(): | 288 for pending in self.queue.iterate(): |
| 290 why_not = pending.why_not() | 289 why_not = pending.why_not() |
| 291 if why_nots[pending.issue] != why_not: | 290 if why_nots[pending.issue] != why_not: |
| 292 self.context.status.send( | 291 self.context.status.send( |
| 293 pending, | 292 pending, |
| 294 {'verification': 'why not', | 293 {'verification': 'why not', |
| 295 'payload': {'message': why_not}}) | 294 'payload': {'message': why_not}}) |
| 296 | 295 |
| 297 | 296 |
| 298 def scan_results(self): | 297 def scan_results(self): |
| 299 """Scans pending commits that can be committed or discarded.""" | 298 """Scans pending commits that can be committed or discarded.""" |
| 300 for pending in self.queue.iterate(): | 299 for pending in self.queue.iterate(): |
| 301 state = pending.get_state() | 300 state = pending.get_state() |
| 302 if state == base.FAILED: | 301 if state == base.FAILED: |
| 303 message = pending.error_message() | 302 self._discard_pending( |
| 304 if not message: | 303 pending, pending.error_message() or self.FAILED_NO_MESSAGE) |
| 305 message = 'scan_results(FAILED): ' + self.FAILED_NO_MESSAGE | |
| 306 self._discard_pending(pending, message) | |
| 307 elif state == base.SUCCEEDED: | 304 elif state == base.SUCCEEDED: |
| 308 if self._throttle(pending): | 305 if self._throttle(pending): |
| 309 continue | 306 continue |
| 310 try: | 307 try: |
| 311 # Runs checks. It's be nice to run the test before the postpone, | 308 # Runs checks. It's be nice to run the test before the postpone, |
| 312 # especially if the tree is closed for a long moment but at the same | 309 # especially if the tree is closed for a long moment but at the same |
| 313 # time it would keep fetching the rietveld status constantly. | 310 # time it would keep fetching the rietveld status constantly. |
| 314 self._last_minute_checks(pending) | 311 self._last_minute_checks(pending) |
| 315 self.context.status.send( | 312 self.context.status.send( |
| 316 pending, | 313 pending, |
| 317 {'verification': 'why not', | 314 {'verification': 'why not', |
| 318 'payload': {'message': ''}}) | 315 'payload': {'message': ''}}) |
| 319 | 316 |
| 320 self._commit_patch(pending) | 317 self._commit_patch(pending) |
| 321 except base.DiscardPending as e: | 318 except base.DiscardPending as e: |
| 322 message = e.status | 319 self._discard_pending(e.pending, e.status) |
| 323 if not message: | |
| 324 message = 'scan_results(discard): ' + self.FAILED_NO_MESSAGE | |
| 325 self._discard_pending(e.pending, message) | |
| 326 except Exception as e: | 320 except Exception as e: |
| 327 message = 'scan_result(Exception): ' + self.INTERNAL_EXCEPTION | 321 self._discard_pending(pending, self.INTERNAL_EXCEPTION) |
| 328 self._discard_pending(pending, message) | |
| 329 raise | 322 raise |
| 330 else: | 323 else: |
| 331 # When state is IGNORED, we need to keep this issue so it's not fetched | 324 # When state is IGNORED, we need to keep this issue so it's not fetched |
| 332 # another time but we can't discard it since we don't want to remove the | 325 # another time but we can't discard it since we don't want to remove the |
| 333 # commit bit for another project hosted on the same code review | 326 # commit bit for another project hosted on the same code review |
| 334 # instance. | 327 # instance. |
| 335 assert state in (base.PROCESSING, base.IGNORED) | 328 assert state in (base.PROCESSING, base.IGNORED) |
| 336 | 329 |
| 337 def _verify_pending(self, pending): | 330 def _verify_pending(self, pending): |
| 338 """Initiates all the verifiers on a pending change.""" | 331 """Initiates all the verifiers on a pending change.""" |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 423 pending, | 416 pending, |
| 424 'List of reviewers changed. %s did a drive-by without LGTM\'ing!' % | 417 'List of reviewers changed. %s did a drive-by without LGTM\'ing!' % |
| 425 ','.join(drivers_by)) | 418 ','.join(drivers_by)) |
| 426 if pending.patchset != pending_data['patchsets'][-1]: | 419 if pending.patchset != pending_data['patchsets'][-1]: |
| 427 raise base.DiscardPending(pending, | 420 raise base.DiscardPending(pending, |
| 428 'Commit queue failed due to new patchset.') | 421 'Commit queue failed due to new patchset.') |
| 429 | 422 |
| 430 def _discard_pending(self, pending, message): | 423 def _discard_pending(self, pending, message): |
| 431 """Discards a pending commit. Attach an optional message to the review.""" | 424 """Discards a pending commit. Attach an optional message to the review.""" |
| 432 logging.debug('_discard_pending(%s, %s)', pending.issue, message) | 425 logging.debug('_discard_pending(%s, %s)', pending.issue, message) |
| 433 msg = message or self.FAILED_NO_MESSAGE | |
| 434 try: | 426 try: |
| 435 try: | 427 try: |
| 436 if pending.get_state() != base.IGNORED: | 428 if pending.get_state() != base.IGNORED: |
| 437 self.context.rietveld.set_flag( | 429 self.context.rietveld.set_flag( |
| 438 pending.issue, pending.patchset, 'commit', False) | 430 pending.issue, pending.patchset, 'commit', False) |
| 439 except urllib2.HTTPError as e: | 431 except urllib2.HTTPError as e: |
| 440 logging.error( | 432 logging.error( |
| 441 'Failed to set the flag to False for %s with message %s' % ( | 433 'Failed to set the flag to False for %s with message %s' % ( |
| 442 pending.pending_name(), msg)) | 434 pending.pending_name(), message)) |
| 443 traceback.print_stack() | 435 traceback.print_stack() |
| 444 logging.error(str(e)) | 436 logging.error(str(e)) |
| 445 errors.send_stack(e) | 437 errors.send_stack(e) |
| 446 try: | 438 if message: |
| 447 self.context.rietveld.add_comment(pending.issue, msg) | 439 try: |
| 448 except urllib2.HTTPError as e: | 440 self.context.rietveld.add_comment(pending.issue, message) |
| 449 logging.error( | 441 except urllib2.HTTPError as e: |
| 450 'Failed to add comment for %s with message %s' % ( | 442 logging.error( |
| 451 pending.pending_name(), msg)) | 443 'Failed to add comment for %s with message %s' % ( |
| 452 traceback.print_stack() | 444 pending.pending_name(), message)) |
| 453 errors.send_stack(e) | 445 traceback.print_stack() |
| 454 self.context.status.send( | 446 errors.send_stack(e) |
| 455 pending, | 447 self.context.status.send( |
| 456 { 'verification': 'abort', | 448 pending, |
| 457 'payload': { | 449 { 'verification': 'abort', |
| 458 'output': msg }}) | 450 'payload': { |
| 451 'output': message }}) |
| 459 finally: | 452 finally: |
| 460 # Most importantly, remove the PendingCommit from the queue. | 453 # Most importantly, remove the PendingCommit from the queue. |
| 461 self.queue.remove(pending.issue) | 454 self.queue.remove(pending.issue) |
| 462 | 455 |
| 463 def _commit_patch(self, pending): | 456 def _commit_patch(self, pending): |
| 464 """Commits the pending patch to the repository. | 457 """Commits the pending patch to the repository. |
| 465 | 458 |
| 466 Do the checkout and applies the patch. | 459 Do the checkout and applies the patch. |
| 467 """ | 460 """ |
| 468 try: | 461 try: |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 527 except ( | 520 except ( |
| 528 checkout.PatchApplicationFailed, patch.UnsupportedPatchFormat) as e: | 521 checkout.PatchApplicationFailed, patch.UnsupportedPatchFormat) as e: |
| 529 raise base.DiscardPending(pending, str(e)) | 522 raise base.DiscardPending(pending, str(e)) |
| 530 except subprocess2.CalledProcessError as e: | 523 except subprocess2.CalledProcessError as e: |
| 531 stdout = getattr(e, 'stdout', None) | 524 stdout = getattr(e, 'stdout', None) |
| 532 out = 'Failed to apply the patch.' | 525 out = 'Failed to apply the patch.' |
| 533 if stdout: | 526 if stdout: |
| 534 out += '\n%s' % stdout | 527 out += '\n%s' % stdout |
| 535 raise base.DiscardPending(pending, out) | 528 raise base.DiscardPending(pending, out) |
| 536 except base.DiscardPending as e: | 529 except base.DiscardPending as e: |
| 537 message = e.status | 530 self._discard_pending(e.pending, e.status) |
| 538 if not message: | |
| 539 message = '_commit_patch: ' + self.FAILED_NO_MESSAGE | |
| 540 self._discard_pending(e.pending, message) | |
| 541 | 531 |
| 542 def _throttle(self, pending): | 532 def _throttle(self, pending): |
| 543 """Returns True if a commit should be delayed.""" | 533 """Returns True if a commit should be delayed.""" |
| 544 if pending.postpone(): | 534 if pending.postpone(): |
| 545 self.context.status.send( | 535 self.context.status.send( |
| 546 pending, | 536 pending, |
| 547 {'verification': 'why not', | 537 {'verification': 'why not', |
| 548 'payload': { | 538 'payload': { |
| 549 'message': pending.why_not()}}) | 539 'message': pending.why_not()}}) |
| 550 return True | 540 return True |
| (...skipping 19 matching lines...) Expand all Loading... |
| 570 """Loads the commit queue state from a JSON file.""" | 560 """Loads the commit queue state from a JSON file.""" |
| 571 self.queue = model.load_from_json_file(filename) | 561 self.queue = model.load_from_json_file(filename) |
| 572 | 562 |
| 573 def save(self, filename): | 563 def save(self, filename): |
| 574 """Save the commit queue state in a simple JSON file.""" | 564 """Save the commit queue state in a simple JSON file.""" |
| 575 model.save_to_json_file(filename, self.queue) | 565 model.save_to_json_file(filename, self.queue) |
| 576 | 566 |
| 577 def close(self): | 567 def close(self): |
| 578 """Close all the active pending manager items.""" | 568 """Close all the active pending manager items.""" |
| 579 self.context.status.close() | 569 self.context.status.close() |
| OLD | NEW |